[Statepoints] Support for "patchable" statepoints.

Summary:
This change adds two new parameters to the statepoint intrinsic, `i64 id`
and `i32 num_patch_bytes`.  `id` gets propagated to the ID field
in the generated StackMap section.  If the `num_patch_bytes` is
non-zero then the statepoint is lowered to `num_patch_bytes` bytes of
nops instead of a call (the spill and reload code remains unchanged).
A non-zero `num_patch_bytes` is useful in situations where a language
runtime requires complete control over how a call is lowered.

This change brings statepoints one step closer to patchpoints.  With
some additional work (that is not part of this patch) it should be
possible to get rid of `TargetOpcode::STATEPOINT` altogether.

PlaceSafepoints generates `statepoint` wrappers with `id` set to
`0xABCDEF00` (the old default value for the ID reported in the stackmap)
and `num_patch_bytes` set to `0`.  This can be made more sophisticated
later.

Reviewers: reames, pgavlin, swaroop.sridhar, AndyAyers

Subscribers: llvm-commits

Differential Revision: http://reviews.llvm.org/D9546

llvm-svn: 237214

6 files changed, 123 insertions, 88 deletions

diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 29f4b0ae182..7a809f4239f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -583,11 +583,6 @@ void SelectionDAGBuilder::LowerStatepoint(
   // Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
   // nodes with all the appropriate arguments and return values.
 
-  // TODO: Currently, all of these operands are being marked as read/write in
-  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
-  // and flags to be read-only.
-  SmallVector<SDValue, 40> Ops;
-
   // Call Node: Chain, Target, {Args}, RegMask, [Glue]
   SDValue Chain = CallNode->getOperand(0);
 
@@ -634,6 +629,16 @@ void SelectionDAGBuilder::LowerStatepoint(
     Glue = GCTransitionStart.getValue(1);
   }
 
+  // TODO: Currently, all of these operands are being marked as read/write in
+  // PrologEpilougeInserter.cpp, we should special case the VMState arguments
+  // and flags to be read-only.
+  SmallVector<SDValue, 40> Ops;
+
+  // Add the <id> and <numBytes> constants.
+  Ops.push_back(DAG.getTargetConstant(ISP.getID(), getCurSDLoc(), MVT::i64));
+  Ops.push_back(
+      DAG.getTargetConstant(ISP.getNumPatchBytes(), getCurSDLoc(), MVT::i32));
+
   // Calculate and push starting position of vmstate arguments
   // Get number of arguments incoming directly into call node
   unsigned NumCallRegArgs =
@@ -657,7 +662,7 @@ void SelectionDAGBuilder::LowerStatepoint(
   pushStackMapConstant(Ops, *this, CS.getCallingConv());
 
   // Add a constant argument for the flags
-  uint64_t Flags = cast<ConstantInt>(CS.getArgument(2))->getZExtValue();
+  uint64_t Flags = ISP.getFlags();
   assert(
       ((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0)
           && "unknown flag used");
diff --git a/llvm/lib/CodeGen/StackMaps.cpp b/llvm/lib/CodeGen/StackMaps.cpp
index a7868329e4d..b5a961dfec1 100644
--- a/llvm/lib/CodeGen/StackMaps.cpp
+++ b/llvm/lib/CodeGen/StackMaps.cpp
@@ -370,9 +370,8 @@ void StackMaps::recordStatepoint(const MachineInstr &MI) {
   // Record all the deopt and gc operands (they're contiguous and run from the
   // initial index to the end of the operand list)
   const unsigned StartIdx = opers.getVarIdx();
-  recordStackMapOpers(MI, 0xABCDEF00,
-                      MI.operands_begin() + StartIdx, MI.operands_end(),
-                      false);
+  recordStackMapOpers(MI, opers.getID(), MI.operands_begin() + StartIdx,
+                      MI.operands_end(), false);
 }
 
 /// Emit the stackmap header.
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 87fca8c8610..a0b61494e26 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -245,12 +245,13 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(unsigned Id,
   return createCallHelper(TheFn, Ops, this, Name);
 }
 
-static std::vector<Value *> getStatepointArgs(IRBuilderBase &B,
-                                              Value *ActualCallee,
-                                              ArrayRef<Value *> CallArgs,
-                                              ArrayRef<Value *> DeoptArgs,
-                                              ArrayRef<Value *> GCArgs) {
+static std::vector<Value *>
+getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
+                  Value *ActualCallee, ArrayRef<Value *> CallArgs,
+                  ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs) {
   std::vector<Value *> Args;
+  Args.push_back(B.getInt64(ID));
+  Args.push_back(B.getInt32(NumPatchBytes));
   Args.push_back(ActualCallee);
   Args.push_back(B.getInt32(CallArgs.size()));
   Args.push_back(B.getInt32((unsigned)StatepointFlags::None));
@@ -263,11 +264,10 @@ static std::vector<Value *> getStatepointArgs(IRBuilderBase &B,
   return Args;
 }
 
-CallInst *IRBuilderBase::CreateGCStatepointCall(Value *ActualCallee,
-                                                ArrayRef<Value *> CallArgs,
-                                                ArrayRef<Value *> DeoptArgs,
-                                                ArrayRef<Value *> GCArgs,
-                                                const Twine &Name) {
+CallInst *IRBuilderBase::CreateGCStatepointCall(
+    uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
+    ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
+    ArrayRef<Value *> GCArgs, const Twine &Name) {
   // Extract out the type of the callee.
   PointerType *FuncPtrType = cast<PointerType>(ActualCallee->getType());
   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
@@ -280,25 +280,25 @@ CallInst *IRBuilderBase::CreateGCStatepointCall(Value *ActualCallee,
     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
                               ArgTypes);
 
-  std::vector<llvm::Value *> Args =
-      getStatepointArgs(*this, ActualCallee, CallArgs, DeoptArgs, GCArgs);
+  std::vector<llvm::Value *> Args = getStatepointArgs(
+      *this, ID, NumPatchBytes, ActualCallee, CallArgs, DeoptArgs, GCArgs);
   return createCallHelper(FnStatepoint, Args, this, Name);
 }
 
-CallInst *IRBuilderBase::CreateGCStatepointCall(Value *ActualCallee,
-                                                ArrayRef<Use> CallArgs,
-                                                ArrayRef<Value *> DeoptArgs,
-                                                ArrayRef<Value *> GCArgs,
-                                                const Twine &Name) {
+CallInst *IRBuilderBase::CreateGCStatepointCall(
+    uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
+    ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
+    ArrayRef<Value *> GCArgs, const Twine &Name) {
   std::vector<Value *> VCallArgs;
   for (auto &U : CallArgs)
     VCallArgs.push_back(U.get());
-  return CreateGCStatepointCall(ActualCallee, VCallArgs, DeoptArgs, GCArgs,
-                                Name);
+  return CreateGCStatepointCall(ID, NumPatchBytes, ActualCallee, VCallArgs,
+                                DeoptArgs, GCArgs, Name);
 }
 
 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
-    Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
+    uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
+    BasicBlock *NormalDest, BasicBlock *UnwindDest,
     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
     ArrayRef<Value *> GCArgs, const Twine &Name) {
   // Extract out the type of the callee.
@@ -311,21 +311,22 @@ InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
   Function *FnStatepoint = Intrinsic::getDeclaration(
       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
 
-  std::vector<llvm::Value *> Args =
-      getStatepointArgs(*this, ActualInvokee, InvokeArgs, DeoptArgs, GCArgs);
+  std::vector<llvm::Value *> Args = getStatepointArgs(
+      *this, ID, NumPatchBytes, ActualInvokee, InvokeArgs, DeoptArgs, GCArgs);
   return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, this,
                             Name);
 }
 
 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
-    Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
-    ArrayRef<Use> InvokeArgs, ArrayRef<Value *> DeoptArgs,
-    ArrayRef<Value *> GCArgs, const Twine &Name) {
+    uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
+    BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
+    ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
   std::vector<Value *> VCallArgs;
   for (auto &U : InvokeArgs)
     VCallArgs.push_back(U.get());
-  return CreateGCStatepointInvoke(ActualInvokee, NormalDest, UnwindDest,
-                                  VCallArgs, DeoptArgs, GCArgs, Name);
+  return CreateGCStatepointInvoke(ID, NumPatchBytes, ActualInvokee, NormalDest,
+                                  UnwindDest, VCallArgs, DeoptArgs, GCArgs,
+                                  Name);
 }
 
 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index cdb78d5db6e..003b6064cbe 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1499,13 +1499,34 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
          "reordering restrictions required by safepoint semantics",
          &CI);
 
-  const Value *Target = CS.getArgument(0);
+  const Value *IDV = CS.getArgument(0);
+  Assert(isa<ConstantInt>(IDV), "gc.statepoint ID must be a constant integer",
+         &CI);
+
+  const Value *NumPatchBytesV = CS.getArgument(1);
+  Assert(isa<ConstantInt>(NumPatchBytesV),
+         "gc.statepoint number of patchable bytes must be a constant integer",
+         &CI);
+  const uint64_t NumPatchBytes =
+      cast<ConstantInt>(NumPatchBytesV)->getSExtValue();
+  assert(isInt<32>(NumPatchBytes) && "NumPatchBytesV is an i32!");
+  Assert(NumPatchBytes >= 0, "gc.statepoint number of patchable bytes must be "
+                             "positive",
+         &CI);
+
+  const Value *Target = CS.getArgument(2);
   const PointerType *PT = dyn_cast<PointerType>(Target->getType());
   Assert(PT && PT->getElementType()->isFunctionTy(),
          "gc.statepoint callee must be of function pointer type", &CI, Target);
   FunctionType *TargetFuncType = cast<FunctionType>(PT->getElementType());
 
-  const Value *NumCallArgsV = CS.getArgument(1);
+  if (NumPatchBytes)
+    Assert(isa<ConstantPointerNull>(Target->stripPointerCasts()),
+           "gc.statepoint must have null as call target if number of patchable "
+           "bytes is non zero",
+           &CI);
+
+  const Value *NumCallArgsV = CS.getArgument(3);
   Assert(isa<ConstantInt>(NumCallArgsV),
          "gc.statepoint number of arguments to underlying call "
          "must be constant integer",
@@ -1529,7 +1550,7 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
     Assert(NumCallArgs == NumParams,
            "gc.statepoint mismatch in number of call args", &CI);
 
-  const Value *FlagsV = CS.getArgument(2);
+  const Value *FlagsV = CS.getArgument(4);
   Assert(isa<ConstantInt>(FlagsV),
          "gc.statepoint flags must be constant integer", &CI);
   const uint64_t Flags = cast<ConstantInt>(FlagsV)->getZExtValue();
@@ -1540,13 +1561,14 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
   // the type of the wrapped callee.
   for (int i = 0; i < NumParams; i++) {
     Type *ParamType = TargetFuncType->getParamType(i);
-    Type *ArgType = CS.getArgument(3+i)->getType();
+    Type *ArgType = CS.getArgument(5 + i)->getType();
     Assert(ArgType == ParamType,
            "gc.statepoint call argument does not match wrapped "
            "function type",
            &CI);
   }
-  const int EndCallArgsInx = 2+NumCallArgs;
+
+  const int EndCallArgsInx = 4 + NumCallArgs;
 
   const Value *NumTransitionArgsV = CS.getArgument(EndCallArgsInx+1);
   Assert(isa<ConstantInt>(NumTransitionArgsV),
@@ -1570,7 +1592,7 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
          &CI);
 
   const int ExpectedNumArgs =
-      5 + NumCallArgs + NumTransitionArgs + NumDeoptArgs;
+      7 + NumCallArgs + NumTransitionArgs + NumDeoptArgs;
   Assert(ExpectedNumArgs <= (int)CS.arg_size(),
          "gc.statepoint too few arguments according to length fields", &CI);
 
@@ -3289,7 +3311,7 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
            CI.getArgOperand(0));
 
     // Assert that result type matches wrapped callee.
-    const Value *Target = StatepointCS.getArgument(0);
+    const Value *Target = StatepointCS.getArgument(2);
     const PointerType *PT = cast<PointerType>(Target->getType());
     const FunctionType *TargetFuncType =
       cast<FunctionType>(PT->getElementType());
@@ -3356,18 +3378,19 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
     // section of the statepoint's argument
     Assert(StatepointCS.arg_size() > 0,
            "gc.statepoint: insufficient arguments");
-    Assert(isa<ConstantInt>(StatepointCS.getArgument(1)),
+    Assert(isa<ConstantInt>(StatepointCS.getArgument(3)),
            "gc.statement: number of call arguments must be constant integer");
     const unsigned NumCallArgs =
-      cast<ConstantInt>(StatepointCS.getArgument(1))->getZExtValue();
-    Assert(StatepointCS.arg_size() > NumCallArgs+3,
+        cast<ConstantInt>(StatepointCS.getArgument(3))->getZExtValue();
+    Assert(StatepointCS.arg_size() > NumCallArgs + 5,
            "gc.statepoint: mismatch in number of call arguments");
-    Assert(isa<ConstantInt>(StatepointCS.getArgument(NumCallArgs+3)),
+    Assert(isa<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 5)),
            "gc.statepoint: number of transition arguments must be "
            "a constant integer");
     const int NumTransitionArgs =
-      cast<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 3))->getZExtValue();
-    const int DeoptArgsStart = 2 + NumCallArgs + 1 + NumTransitionArgs + 1;
+        cast<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 5))
+            ->getZExtValue();
+    const int DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1;
     Assert(isa<ConstantInt>(StatepointCS.getArgument(DeoptArgsStart)),
            "gc.statepoint: number of deoptimization arguments must be "
            "a constant integer");
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 273444cccc0..d55e9f656b0 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -812,43 +812,50 @@ void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
                                     X86MCInstLower &MCIL) {
   assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
 
-  // Lower call target and choose correct opcode
-  const MachineOperand &CallTarget = StatepointOpers(&MI).getCallTarget();
-  MCOperand CallTargetMCOp;
-  unsigned CallOpcode;
-  switch (CallTarget.getType()) {
-  case MachineOperand::MO_GlobalAddress:
-  case MachineOperand::MO_ExternalSymbol:
-    CallTargetMCOp = MCIL.LowerSymbolOperand(
-        CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
-    CallOpcode = X86::CALL64pcrel32;
-    // Currently, we only support relative addressing with statepoints.
-    // Otherwise, we'll need a scratch register to hold the target
-    // address.  You'll fail asserts during load & relocation if this
-    // symbol is to far away. (TODO: support non-relative addressing)
-    break;
-  case MachineOperand::MO_Immediate:
-    CallTargetMCOp = MCOperand::CreateImm(CallTarget.getImm());
-    CallOpcode = X86::CALL64pcrel32;
-    // Currently, we only support relative addressing with statepoints.
-    // Otherwise, we'll need a scratch register to hold the target
-    // immediate.  You'll fail asserts during load & relocation if this
-    // address is to far away. (TODO: support non-relative addressing)
-    break;
-  case MachineOperand::MO_Register:
-    CallTargetMCOp = MCOperand::CreateReg(CallTarget.getReg());
-    CallOpcode = X86::CALL64r;
-    break;
-  default:
-    llvm_unreachable("Unsupported operand type in statepoint call target");
-    break;
-  }
+  StatepointOpers SOpers(&MI);
 
-  // Emit call
-  MCInst CallInst;
-  CallInst.setOpcode(CallOpcode);
-  CallInst.addOperand(CallTargetMCOp);
-  OutStreamer->EmitInstruction(CallInst, getSubtargetInfo());
+  if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
+    EmitNops(*OutStreamer, PatchBytes, Subtarget->is64Bit(),
+             getSubtargetInfo());
+  } else {
+    // Lower call target and choose correct opcode
+    const MachineOperand &CallTarget = SOpers.getCallTarget();
+    MCOperand CallTargetMCOp;
+    unsigned CallOpcode;
+    switch (CallTarget.getType()) {
+    case MachineOperand::MO_GlobalAddress:
+    case MachineOperand::MO_ExternalSymbol:
+      CallTargetMCOp = MCIL.LowerSymbolOperand(
+          CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
+      CallOpcode = X86::CALL64pcrel32;
+      // Currently, we only support relative addressing with statepoints.
+      // Otherwise, we'll need a scratch register to hold the target
+      // address.  You'll fail asserts during load & relocation if this
+      // symbol is to far away. (TODO: support non-relative addressing)
+      break;
+    case MachineOperand::MO_Immediate:
+      CallTargetMCOp = MCOperand::CreateImm(CallTarget.getImm());
+      CallOpcode = X86::CALL64pcrel32;
+      // Currently, we only support relative addressing with statepoints.
+      // Otherwise, we'll need a scratch register to hold the target
+      // immediate.  You'll fail asserts during load & relocation if this
+      // address is to far away. (TODO: support non-relative addressing)
+      break;
+    case MachineOperand::MO_Register:
+      CallTargetMCOp = MCOperand::CreateReg(CallTarget.getReg());
+      CallOpcode = X86::CALL64r;
+      break;
+    default:
+      llvm_unreachable("Unsupported operand type in statepoint call target");
+      break;
+    }
+
+    // Emit call
+    MCInst CallInst;
+    CallInst.setOpcode(CallOpcode);
+    CallInst.addOperand(CallTargetMCOp);
+    OutStreamer->EmitInstruction(CallInst, getSubtargetInfo());
+  }
 
   // Record our statepoint node in the same section used by STACKMAP
   // and PATCHPOINT
diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
index ff3d67a6548..a0377a54683 100644
--- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
+++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
@@ -904,8 +904,8 @@ static Value *ReplaceWithStatepoint(const CallSite &CS, /* to replace */
   if (CS.isCall()) {
     CallInst *ToReplace = cast<CallInst>(CS.getInstruction());
     CallInst *Call = Builder.CreateGCStatepointCall(
-        CS.getCalledValue(), makeArrayRef(CS.arg_begin(), CS.arg_end()), None,
-        None, "safepoint_token");
+        0xABCDEF00, 0, CS.getCalledValue(), makeArrayRef(CS.arg_begin(), CS.arg_end()),
+        None, None, "safepoint_token");
     Call->setTailCall(ToReplace->isTailCall());
     Call->setCallingConv(ToReplace->getCallingConv());
 
@@ -933,7 +933,7 @@ static Value *ReplaceWithStatepoint(const CallSite &CS, /* to replace */
     // original block.
     Builder.SetInsertPoint(ToReplace->getParent());
     InvokeInst *Invoke = Builder.CreateGCStatepointInvoke(
-        CS.getCalledValue(), ToReplace->getNormalDest(),
+        0xABCDEF00, 0, CS.getCalledValue(), ToReplace->getNormalDest(),
         ToReplace->getUnwindDest(), makeArrayRef(CS.arg_begin(), CS.arg_end()),
         Builder.getInt32(0), None, "safepoint_token");