summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp17
-rw-r--r--llvm/lib/CodeGen/StackMaps.cpp5
-rw-r--r--llvm/lib/IR/IRBuilder.cpp55
-rw-r--r--llvm/lib/IR/Verifier.cpp49
-rw-r--r--llvm/lib/Target/X86/X86MCInstLower.cpp79
-rw-r--r--llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp6
6 files changed, 123 insertions, 88 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 29f4b0ae182..7a809f4239f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -583,11 +583,6 @@ void SelectionDAGBuilder::LowerStatepoint(
// Construct the actual GC_TRANSITION_START, STATEPOINT, and GC_TRANSITION_END
// nodes with all the appropriate arguments and return values.
- // TODO: Currently, all of these operands are being marked as read/write in
- // PrologEpilougeInserter.cpp, we should special case the VMState arguments
- // and flags to be read-only.
- SmallVector<SDValue, 40> Ops;
-
// Call Node: Chain, Target, {Args}, RegMask, [Glue]
SDValue Chain = CallNode->getOperand(0);
@@ -634,6 +629,16 @@ void SelectionDAGBuilder::LowerStatepoint(
Glue = GCTransitionStart.getValue(1);
}
+ // TODO: Currently, all of these operands are being marked as read/write in
+ // PrologEpilougeInserter.cpp, we should special case the VMState arguments
+ // and flags to be read-only.
+ SmallVector<SDValue, 40> Ops;
+
+ // Add the <id> and <numBytes> constants.
+ Ops.push_back(DAG.getTargetConstant(ISP.getID(), getCurSDLoc(), MVT::i64));
+ Ops.push_back(
+ DAG.getTargetConstant(ISP.getNumPatchBytes(), getCurSDLoc(), MVT::i32));
+
// Calculate and push starting position of vmstate arguments
// Get number of arguments incoming directly into call node
unsigned NumCallRegArgs =
@@ -657,7 +662,7 @@ void SelectionDAGBuilder::LowerStatepoint(
pushStackMapConstant(Ops, *this, CS.getCallingConv());
// Add a constant argument for the flags
- uint64_t Flags = cast<ConstantInt>(CS.getArgument(2))->getZExtValue();
+ uint64_t Flags = ISP.getFlags();
assert(
((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0)
&& "unknown flag used");
diff --git a/llvm/lib/CodeGen/StackMaps.cpp b/llvm/lib/CodeGen/StackMaps.cpp
index a7868329e4d..b5a961dfec1 100644
--- a/llvm/lib/CodeGen/StackMaps.cpp
+++ b/llvm/lib/CodeGen/StackMaps.cpp
@@ -370,9 +370,8 @@ void StackMaps::recordStatepoint(const MachineInstr &MI) {
// Record all the deopt and gc operands (they're contiguous and run from the
// initial index to the end of the operand list)
const unsigned StartIdx = opers.getVarIdx();
- recordStackMapOpers(MI, 0xABCDEF00,
- MI.operands_begin() + StartIdx, MI.operands_end(),
- false);
+ recordStackMapOpers(MI, opers.getID(), MI.operands_begin() + StartIdx,
+ MI.operands_end(), false);
}
/// Emit the stackmap header.
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 87fca8c8610..a0b61494e26 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -245,12 +245,13 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(unsigned Id,
return createCallHelper(TheFn, Ops, this, Name);
}
-static std::vector<Value *> getStatepointArgs(IRBuilderBase &B,
- Value *ActualCallee,
- ArrayRef<Value *> CallArgs,
- ArrayRef<Value *> DeoptArgs,
- ArrayRef<Value *> GCArgs) {
+static std::vector<Value *>
+getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
+ Value *ActualCallee, ArrayRef<Value *> CallArgs,
+ ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs) {
std::vector<Value *> Args;
+ Args.push_back(B.getInt64(ID));
+ Args.push_back(B.getInt32(NumPatchBytes));
Args.push_back(ActualCallee);
Args.push_back(B.getInt32(CallArgs.size()));
Args.push_back(B.getInt32((unsigned)StatepointFlags::None));
@@ -263,11 +264,10 @@ static std::vector<Value *> getStatepointArgs(IRBuilderBase &B,
return Args;
}
-CallInst *IRBuilderBase::CreateGCStatepointCall(Value *ActualCallee,
- ArrayRef<Value *> CallArgs,
- ArrayRef<Value *> DeoptArgs,
- ArrayRef<Value *> GCArgs,
- const Twine &Name) {
+CallInst *IRBuilderBase::CreateGCStatepointCall(
+ uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
+ ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
+ ArrayRef<Value *> GCArgs, const Twine &Name) {
// Extract out the type of the callee.
PointerType *FuncPtrType = cast<PointerType>(ActualCallee->getType());
assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
@@ -280,25 +280,25 @@ CallInst *IRBuilderBase::CreateGCStatepointCall(Value *ActualCallee,
Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
ArgTypes);
- std::vector<llvm::Value *> Args =
- getStatepointArgs(*this, ActualCallee, CallArgs, DeoptArgs, GCArgs);
+ std::vector<llvm::Value *> Args = getStatepointArgs(
+ *this, ID, NumPatchBytes, ActualCallee, CallArgs, DeoptArgs, GCArgs);
return createCallHelper(FnStatepoint, Args, this, Name);
}
-CallInst *IRBuilderBase::CreateGCStatepointCall(Value *ActualCallee,
- ArrayRef<Use> CallArgs,
- ArrayRef<Value *> DeoptArgs,
- ArrayRef<Value *> GCArgs,
- const Twine &Name) {
+CallInst *IRBuilderBase::CreateGCStatepointCall(
+ uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
+ ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
+ ArrayRef<Value *> GCArgs, const Twine &Name) {
std::vector<Value *> VCallArgs;
for (auto &U : CallArgs)
VCallArgs.push_back(U.get());
- return CreateGCStatepointCall(ActualCallee, VCallArgs, DeoptArgs, GCArgs,
- Name);
+ return CreateGCStatepointCall(ID, NumPatchBytes, ActualCallee, VCallArgs,
+ DeoptArgs, GCArgs, Name);
}
InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
- Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
+ uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
+ BasicBlock *NormalDest, BasicBlock *UnwindDest,
ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
ArrayRef<Value *> GCArgs, const Twine &Name) {
// Extract out the type of the callee.
@@ -311,21 +311,22 @@ InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
Function *FnStatepoint = Intrinsic::getDeclaration(
M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
- std::vector<llvm::Value *> Args =
- getStatepointArgs(*this, ActualInvokee, InvokeArgs, DeoptArgs, GCArgs);
+ std::vector<llvm::Value *> Args = getStatepointArgs(
+ *this, ID, NumPatchBytes, ActualInvokee, InvokeArgs, DeoptArgs, GCArgs);
return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, this,
Name);
}
InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
- Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
- ArrayRef<Use> InvokeArgs, ArrayRef<Value *> DeoptArgs,
- ArrayRef<Value *> GCArgs, const Twine &Name) {
+ uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
+ BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
+ ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
std::vector<Value *> VCallArgs;
for (auto &U : InvokeArgs)
VCallArgs.push_back(U.get());
- return CreateGCStatepointInvoke(ActualInvokee, NormalDest, UnwindDest,
- VCallArgs, DeoptArgs, GCArgs, Name);
+ return CreateGCStatepointInvoke(ID, NumPatchBytes, ActualInvokee, NormalDest,
+ UnwindDest, VCallArgs, DeoptArgs, GCArgs,
+ Name);
}
CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index cdb78d5db6e..003b6064cbe 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1499,13 +1499,34 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
"reordering restrictions required by safepoint semantics",
&CI);
- const Value *Target = CS.getArgument(0);
+ const Value *IDV = CS.getArgument(0);
+ Assert(isa<ConstantInt>(IDV), "gc.statepoint ID must be a constant integer",
+ &CI);
+
+ const Value *NumPatchBytesV = CS.getArgument(1);
+ Assert(isa<ConstantInt>(NumPatchBytesV),
+ "gc.statepoint number of patchable bytes must be a constant integer",
+ &CI);
+ const uint64_t NumPatchBytes =
+ cast<ConstantInt>(NumPatchBytesV)->getSExtValue();
+ assert(isInt<32>(NumPatchBytes) && "NumPatchBytesV is an i32!");
+ Assert(NumPatchBytes >= 0, "gc.statepoint number of patchable bytes must be "
+ "positive",
+ &CI);
+
+ const Value *Target = CS.getArgument(2);
const PointerType *PT = dyn_cast<PointerType>(Target->getType());
Assert(PT && PT->getElementType()->isFunctionTy(),
"gc.statepoint callee must be of function pointer type", &CI, Target);
FunctionType *TargetFuncType = cast<FunctionType>(PT->getElementType());
- const Value *NumCallArgsV = CS.getArgument(1);
+ if (NumPatchBytes)
+ Assert(isa<ConstantPointerNull>(Target->stripPointerCasts()),
+ "gc.statepoint must have null as call target if number of patchable "
+ "bytes is non zero",
+ &CI);
+
+ const Value *NumCallArgsV = CS.getArgument(3);
Assert(isa<ConstantInt>(NumCallArgsV),
"gc.statepoint number of arguments to underlying call "
"must be constant integer",
@@ -1529,7 +1550,7 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
Assert(NumCallArgs == NumParams,
"gc.statepoint mismatch in number of call args", &CI);
- const Value *FlagsV = CS.getArgument(2);
+ const Value *FlagsV = CS.getArgument(4);
Assert(isa<ConstantInt>(FlagsV),
"gc.statepoint flags must be constant integer", &CI);
const uint64_t Flags = cast<ConstantInt>(FlagsV)->getZExtValue();
@@ -1540,13 +1561,14 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
// the type of the wrapped callee.
for (int i = 0; i < NumParams; i++) {
Type *ParamType = TargetFuncType->getParamType(i);
- Type *ArgType = CS.getArgument(3+i)->getType();
+ Type *ArgType = CS.getArgument(5 + i)->getType();
Assert(ArgType == ParamType,
"gc.statepoint call argument does not match wrapped "
"function type",
&CI);
}
- const int EndCallArgsInx = 2+NumCallArgs;
+
+ const int EndCallArgsInx = 4 + NumCallArgs;
const Value *NumTransitionArgsV = CS.getArgument(EndCallArgsInx+1);
Assert(isa<ConstantInt>(NumTransitionArgsV),
@@ -1570,7 +1592,7 @@ void Verifier::VerifyStatepoint(ImmutableCallSite CS) {
&CI);
const int ExpectedNumArgs =
- 5 + NumCallArgs + NumTransitionArgs + NumDeoptArgs;
+ 7 + NumCallArgs + NumTransitionArgs + NumDeoptArgs;
Assert(ExpectedNumArgs <= (int)CS.arg_size(),
"gc.statepoint too few arguments according to length fields", &CI);
@@ -3289,7 +3311,7 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
CI.getArgOperand(0));
// Assert that result type matches wrapped callee.
- const Value *Target = StatepointCS.getArgument(0);
+ const Value *Target = StatepointCS.getArgument(2);
const PointerType *PT = cast<PointerType>(Target->getType());
const FunctionType *TargetFuncType =
cast<FunctionType>(PT->getElementType());
@@ -3356,18 +3378,19 @@ void Verifier::visitIntrinsicFunctionCall(Intrinsic::ID ID, CallInst &CI) {
// section of the statepoint's argument
Assert(StatepointCS.arg_size() > 0,
"gc.statepoint: insufficient arguments");
- Assert(isa<ConstantInt>(StatepointCS.getArgument(1)),
+ Assert(isa<ConstantInt>(StatepointCS.getArgument(3)),
"gc.statement: number of call arguments must be constant integer");
const unsigned NumCallArgs =
- cast<ConstantInt>(StatepointCS.getArgument(1))->getZExtValue();
- Assert(StatepointCS.arg_size() > NumCallArgs+3,
+ cast<ConstantInt>(StatepointCS.getArgument(3))->getZExtValue();
+ Assert(StatepointCS.arg_size() > NumCallArgs + 5,
"gc.statepoint: mismatch in number of call arguments");
- Assert(isa<ConstantInt>(StatepointCS.getArgument(NumCallArgs+3)),
+ Assert(isa<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 5)),
"gc.statepoint: number of transition arguments must be "
"a constant integer");
const int NumTransitionArgs =
- cast<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 3))->getZExtValue();
- const int DeoptArgsStart = 2 + NumCallArgs + 1 + NumTransitionArgs + 1;
+ cast<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 5))
+ ->getZExtValue();
+ const int DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1;
Assert(isa<ConstantInt>(StatepointCS.getArgument(DeoptArgsStart)),
"gc.statepoint: number of deoptimization arguments must be "
"a constant integer");
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 273444cccc0..d55e9f656b0 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -812,43 +812,50 @@ void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
X86MCInstLower &MCIL) {
assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
- // Lower call target and choose correct opcode
- const MachineOperand &CallTarget = StatepointOpers(&MI).getCallTarget();
- MCOperand CallTargetMCOp;
- unsigned CallOpcode;
- switch (CallTarget.getType()) {
- case MachineOperand::MO_GlobalAddress:
- case MachineOperand::MO_ExternalSymbol:
- CallTargetMCOp = MCIL.LowerSymbolOperand(
- CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
- CallOpcode = X86::CALL64pcrel32;
- // Currently, we only support relative addressing with statepoints.
- // Otherwise, we'll need a scratch register to hold the target
- // address. You'll fail asserts during load & relocation if this
- // symbol is to far away. (TODO: support non-relative addressing)
- break;
- case MachineOperand::MO_Immediate:
- CallTargetMCOp = MCOperand::CreateImm(CallTarget.getImm());
- CallOpcode = X86::CALL64pcrel32;
- // Currently, we only support relative addressing with statepoints.
- // Otherwise, we'll need a scratch register to hold the target
- // immediate. You'll fail asserts during load & relocation if this
- // address is to far away. (TODO: support non-relative addressing)
- break;
- case MachineOperand::MO_Register:
- CallTargetMCOp = MCOperand::CreateReg(CallTarget.getReg());
- CallOpcode = X86::CALL64r;
- break;
- default:
- llvm_unreachable("Unsupported operand type in statepoint call target");
- break;
- }
+ StatepointOpers SOpers(&MI);
- // Emit call
- MCInst CallInst;
- CallInst.setOpcode(CallOpcode);
- CallInst.addOperand(CallTargetMCOp);
- OutStreamer->EmitInstruction(CallInst, getSubtargetInfo());
+ if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
+ EmitNops(*OutStreamer, PatchBytes, Subtarget->is64Bit(),
+ getSubtargetInfo());
+ } else {
+ // Lower call target and choose correct opcode
+ const MachineOperand &CallTarget = SOpers.getCallTarget();
+ MCOperand CallTargetMCOp;
+ unsigned CallOpcode;
+ switch (CallTarget.getType()) {
+ case MachineOperand::MO_GlobalAddress:
+ case MachineOperand::MO_ExternalSymbol:
+ CallTargetMCOp = MCIL.LowerSymbolOperand(
+ CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
+ CallOpcode = X86::CALL64pcrel32;
+ // Currently, we only support relative addressing with statepoints.
+ // Otherwise, we'll need a scratch register to hold the target
+ // address. You'll fail asserts during load & relocation if this
+ // symbol is to far away. (TODO: support non-relative addressing)
+ break;
+ case MachineOperand::MO_Immediate:
+ CallTargetMCOp = MCOperand::CreateImm(CallTarget.getImm());
+ CallOpcode = X86::CALL64pcrel32;
+ // Currently, we only support relative addressing with statepoints.
+ // Otherwise, we'll need a scratch register to hold the target
+ // immediate. You'll fail asserts during load & relocation if this
+ // address is to far away. (TODO: support non-relative addressing)
+ break;
+ case MachineOperand::MO_Register:
+ CallTargetMCOp = MCOperand::CreateReg(CallTarget.getReg());
+ CallOpcode = X86::CALL64r;
+ break;
+ default:
+ llvm_unreachable("Unsupported operand type in statepoint call target");
+ break;
+ }
+
+ // Emit call
+ MCInst CallInst;
+ CallInst.setOpcode(CallOpcode);
+ CallInst.addOperand(CallTargetMCOp);
+ OutStreamer->EmitInstruction(CallInst, getSubtargetInfo());
+ }
// Record our statepoint node in the same section used by STACKMAP
// and PATCHPOINT
diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
index ff3d67a6548..a0377a54683 100644
--- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
+++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp
@@ -904,8 +904,8 @@ static Value *ReplaceWithStatepoint(const CallSite &CS, /* to replace */
if (CS.isCall()) {
CallInst *ToReplace = cast<CallInst>(CS.getInstruction());
CallInst *Call = Builder.CreateGCStatepointCall(
- CS.getCalledValue(), makeArrayRef(CS.arg_begin(), CS.arg_end()), None,
- None, "safepoint_token");
+ 0xABCDEF00, 0, CS.getCalledValue(), makeArrayRef(CS.arg_begin(), CS.arg_end()),
+ None, None, "safepoint_token");
Call->setTailCall(ToReplace->isTailCall());
Call->setCallingConv(ToReplace->getCallingConv());
@@ -933,7 +933,7 @@ static Value *ReplaceWithStatepoint(const CallSite &CS, /* to replace */
// original block.
Builder.SetInsertPoint(ToReplace->getParent());
InvokeInst *Invoke = Builder.CreateGCStatepointInvoke(
- CS.getCalledValue(), ToReplace->getNormalDest(),
+ 0xABCDEF00, 0, CS.getCalledValue(), ToReplace->getNormalDest(),
ToReplace->getUnwindDest(), makeArrayRef(CS.arg_begin(), CS.arg_end()),
Builder.getInt32(0), None, "safepoint_token");
OpenPOWER on IntegriCloud