summaryrefslogtreecommitdiffstats
path: root/llvm
diff options
context:
space:
mode:
Diffstat (limited to 'llvm')
-rw-r--r--llvm/include/llvm/CodeGen/StackMaps.h50
-rw-r--r--llvm/include/llvm/Target/Target.td9
-rw-r--r--llvm/include/llvm/Target/TargetFrameLowering.h10
-rw-r--r--llvm/include/llvm/Target/TargetOpcodes.h8
-rw-r--r--llvm/lib/CodeGen/InlineSpiller.cpp3
-rw-r--r--llvm/lib/CodeGen/LocalStackSlotAllocation.cpp1
-rw-r--r--llvm/lib/CodeGen/PrologEpilogInserter.cpp20
-rw-r--r--llvm/lib/CodeGen/StackMaps.cpp12
-rw-r--r--llvm/lib/CodeGen/TargetLoweringBase.cpp8
-rw-r--r--llvm/lib/Target/X86/X86FrameLowering.cpp73
-rw-r--r--llvm/lib/Target/X86/X86FrameLowering.h4
-rw-r--r--llvm/lib/Target/X86/X86ISelLowering.cpp5
-rw-r--r--llvm/lib/Target/X86/X86MCInstLower.cpp64
-rw-r--r--llvm/utils/TableGen/CodeGenTarget.cpp1
14 files changed, 263 insertions, 5 deletions
diff --git a/llvm/include/llvm/CodeGen/StackMaps.h b/llvm/include/llvm/CodeGen/StackMaps.h
index e343980d2ab..4e48afe1400 100644
--- a/llvm/include/llvm/CodeGen/StackMaps.h
+++ b/llvm/include/llvm/CodeGen/StackMaps.h
@@ -81,6 +81,52 @@ public:
unsigned getNextScratchIdx(unsigned StartIdx = 0) const;
};
+/// MI-level Statepoint operands
+///
+/// Statepoint operands take the form:
+/// <num call arguments>, <call target>, [call arguments],
+/// <StackMaps::ConstantOp>, <flags>,
+/// <StackMaps::ConstantOp>, <num other args>, [other args],
+/// [gc values]
+class StatepointOpers {
+private:
+ enum {
+ NCallArgsPos = 0,
+ CallTargetPos = 1
+ };
+
+public:
+ explicit StatepointOpers(const MachineInstr *MI):
+ MI(MI) { }
+
+ /// Get starting index of non call related arguments
+ /// (statepoint flags, vm state and gc state).
+ unsigned getVarIdx() const {
+ return MI->getOperand(NCallArgsPos).getImm() + 2;
+ }
+
+ /// Returns the index of the operand containing the number of non-gc non-call
+ /// arguments.
+ unsigned getNumVMSArgsIdx() const {
+ return getVarIdx() + 3;
+ }
+
+ /// Returns the number of non-gc non-call arguments attached to the
+ /// statepoint. Note that this is the number of arguments, not the number of
+ /// operands required to represent those arguments.
+ unsigned getNumVMSArgs() const {
+ return MI->getOperand(getNumVMSArgsIdx()).getImm();
+ }
+
+ /// Returns the target of the underlying call.
+ const MachineOperand &getCallTarget() const {
+ return MI->getOperand(CallTargetPos);
+ }
+
+private:
+ const MachineInstr *MI;
+};
+
class StackMaps {
public:
struct Location {
@@ -132,6 +178,9 @@ public:
/// \brief Generate a stackmap record for a patchpoint instruction.
void recordPatchPoint(const MachineInstr &MI);
+ /// \brief Generate a stackmap record for a statepoint instruction.
+ void recordStatepoint(const MachineInstr &MI);
+
/// If there is any stack map data, create a stack map section and serialize
/// the map info into it. This clears the stack map data structures
/// afterwards.
@@ -139,7 +188,6 @@ public:
private:
static const char *WSMP;
-
typedef SmallVector<Location, 8> LocationVec;
typedef SmallVector<LiveOutReg, 8> LiveOutVec;
typedef MapVector<uint64_t, uint64_t> ConstantPool;
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index d28ce692a3a..688c03fd7e2 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -845,6 +845,15 @@ def PATCHPOINT : Instruction {
let mayLoad = 1;
let usesCustomInserter = 1;
}
+def STATEPOINT : Instruction {
+ let OutOperandList = (outs);
+ let InOperandList = (ins variable_ops);
+ let usesCustomInserter = 1;
+ let mayLoad = 1;
+ let mayStore = 1;
+ let hasSideEffects = 1;
+ let isCall = 1;
+}
def LOAD_STACK_GUARD : Instruction {
let OutOperandList = (outs ptr_rc:$dst);
let InOperandList = (ins);
diff --git a/llvm/include/llvm/Target/TargetFrameLowering.h b/llvm/include/llvm/Target/TargetFrameLowering.h
index bfddd060179..82c6093d8fb 100644
--- a/llvm/include/llvm/Target/TargetFrameLowering.h
+++ b/llvm/include/llvm/Target/TargetFrameLowering.h
@@ -199,6 +199,16 @@ public:
virtual int getFrameIndexReference(const MachineFunction &MF, int FI,
unsigned &FrameReg) const;
+ /// Same as above, except that the 'base register' will always be RSP, not
+ /// RBP on x86. This is used exclusively for lowering STATEPOINT nodes.
+ /// TODO: This should really be a parameterizable choice.
+ virtual int getFrameIndexReferenceFromSP(const MachineFunction &MF, int FI,
+ unsigned &FrameReg) const {
+ // default to calling normal version, we override this on x86 only
+ llvm_unreachable("unimplemented for non-x86");
+ return 0;
+ }
+
/// processFunctionBeforeCalleeSavedScan - This method is called immediately
/// before PrologEpilogInserter scans the physical registers used to determine
/// what callee saved registers should be spilled. This method is optional.
diff --git a/llvm/include/llvm/Target/TargetOpcodes.h b/llvm/include/llvm/Target/TargetOpcodes.h
index 1fbd2ae09b1..7ab1bc3837f 100644
--- a/llvm/include/llvm/Target/TargetOpcodes.h
+++ b/llvm/include/llvm/Target/TargetOpcodes.h
@@ -110,7 +110,13 @@ enum {
/// to prevent the stack guard value or address from being spilled to the
/// stack should override TargetLowering::emitLoadStackGuardNode and
/// additionally expand this pseudo after register allocation.
- LOAD_STACK_GUARD = 19
+ LOAD_STACK_GUARD = 19,
+
+ /// Call instruction with associated vm state for deoptimization and list
+ /// of live pointers for relocation by the garbage collector. It is
+ /// intended to support garbage collection with fully precise relocating
+ /// collectors and deoptimizations in either the callee or caller.
+ STATEPOINT = 20
};
} // end namespace TargetOpcode
} // end namespace llvm
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index 6a6e15dabff..121a89e0da2 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -1088,7 +1088,8 @@ foldMemoryOperand(ArrayRef<std::pair<MachineInstr*, unsigned> > Ops,
bool WasCopy = MI->isCopy();
unsigned ImpReg = 0;
- bool SpillSubRegs = (MI->getOpcode() == TargetOpcode::PATCHPOINT ||
+ bool SpillSubRegs = (MI->getOpcode() == TargetOpcode::STATEPOINT ||
+ MI->getOpcode() == TargetOpcode::PATCHPOINT ||
MI->getOpcode() == TargetOpcode::STACKMAP);
// TargetInstrInfo::foldMemoryOperand only expects explicit, non-tied
diff --git a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
index 5c5712f37e1..e8bf687a626 100644
--- a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
+++ b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
@@ -291,6 +291,7 @@ bool LocalStackSlotPass::insertFrameReferenceRegisters(MachineFunction &Fn) {
// Debug value, stackmap and patchpoint instructions can't be out of
// range, so they don't need any updates.
if (MI->isDebugValue() ||
+ MI->getOpcode() == TargetOpcode::STATEPOINT ||
MI->getOpcode() == TargetOpcode::STACKMAP ||
MI->getOpcode() == TargetOpcode::PATCHPOINT)
continue;
diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
index 06530b91d67..a19bd96b3f5 100644
--- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp
@@ -797,6 +797,26 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &Fn,
continue;
}
+ // TODO: This code should be commoned with the code for
+ // PATCHPOINT. There's no good reason for the difference in
+ // implementation other than historical accident. The only
+ // remaining difference is the unconditional use of the stack
+ // pointer as the base register.
+ if (MI->getOpcode() == TargetOpcode::STATEPOINT) {
+ assert((!MI->isDebugValue() || i == 0) &&
+ "Frame indicies can only appear as the first operand of a "
+ "DBG_VALUE machine instruction");
+ unsigned Reg;
+ MachineOperand &Offset = MI->getOperand(i + 1);
+ const unsigned refOffset =
+ TFI->getFrameIndexReferenceFromSP(Fn, MI->getOperand(i).getIndex(),
+ Reg);
+
+ Offset.setImm(Offset.getImm() + refOffset);
+ MI->getOperand(i).ChangeToRegister(Reg, false /*isDef*/);
+ continue;
+ }
+
// Some instructions (e.g. inline asm instructions) can have
// multiple frame indices and/or cause eliminateFrameIndex
// to insert more than one instruction. We need the register
diff --git a/llvm/lib/CodeGen/StackMaps.cpp b/llvm/lib/CodeGen/StackMaps.cpp
index d3791c3feb5..aab8d522144 100644
--- a/llvm/lib/CodeGen/StackMaps.cpp
+++ b/llvm/lib/CodeGen/StackMaps.cpp
@@ -286,6 +286,18 @@ void StackMaps::recordPatchPoint(const MachineInstr &MI) {
}
#endif
}
+void StackMaps::recordStatepoint(const MachineInstr &MI) {
+ assert(MI.getOpcode() == TargetOpcode::STATEPOINT &&
+ "expected statepoint");
+
+ StatepointOpers opers(&MI);
+ // Record all the deopt and gc operands (they're contiguous and run from the
+ // initial index to the end of the operand list)
+ const unsigned StartIdx = opers.getVarIdx();
+ recordStackMapOpers(MI, 0xABCDEF00,
+ MI.operands_begin() + StartIdx, MI.operands_end(),
+ false);
+}
/// Emit the stackmap header.
///
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index bd5cd4c8cc2..8805104689e 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -992,8 +992,14 @@ TargetLoweringBase::emitPatchPoint(MachineInstr *MI,
// Add a new memory operand for this FI.
const MachineFrameInfo &MFI = *MF.getFrameInfo();
assert(MFI.getObjectOffset(FI) != -1);
+
+ unsigned Flags = MachineMemOperand::MOLoad;
+ if (MI->getOpcode() == TargetOpcode::STATEPOINT) {
+ Flags |= MachineMemOperand::MOStore;
+ Flags |= MachineMemOperand::MOVolatile;
+ }
MachineMemOperand *MMO = MF.getMachineMemOperand(
- MachinePointerInfo::getFixedStack(FI), MachineMemOperand::MOLoad,
+ MachinePointerInfo::getFixedStack(FI), Flags,
TM.getSubtargetImpl()->getDataLayout()->getPointerSize(),
MFI.getObjectAlignment(FI));
MIB->addMemOperand(MF, MMO);
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index b9920b1b84c..278e4f910d4 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -1135,6 +1135,79 @@ int X86FrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI,
return getFrameIndexOffset(MF, FI);
}
+// Simplified from getFrameIndexOffset keeping only StackPointer cases
+int X86FrameLowering::getFrameIndexOffsetFromSP(const MachineFunction &MF, int FI) const {
+ const X86RegisterInfo *RegInfo =
+ static_cast<const X86RegisterInfo*>(MF.getSubtarget().getRegisterInfo());
+ const MachineFrameInfo *MFI = MF.getFrameInfo();
+ const uint64_t StackSize = MFI->getStackSize(); //not including dynamic realign
+
+ {
+#ifndef NDEBUG
+ // Note: LLVM arranges the stack as:
+ // Args > Saved RetPC (<--FP) > CSRs > dynamic alignment (<--BP)
+ // > "Stack Slots" (<--SP)
+ // We can always address StackSlots from RSP. We can usually (unless
+ // needsStackRealignment) address CSRs from RSP, but sometimes need to
+ // address them from RBP. FixedObjects can be placed anywhere in the stack
+ // frame depending on their specific requirements (i.e. we can actually
+ // refer to arguments to the function which are stored in the *callers*
+ // frame). As a result, THE RESULT OF THIS CALL IS MEANINGLESS FOR CSRs
+ // AND FixedObjects IFF needsStackRealignment or hasVarSizedObject.
+
+ assert(!RegInfo->hasBasePointer(MF) && "we don't handle this case");
+
+ // We don't handle tail calls, and shouldn't be seeing them
+ // either.
+ int TailCallReturnAddrDelta =
+ MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta();
+ assert(!(TailCallReturnAddrDelta < 0) && "we don't handle this case!");
+#endif
+ }
+
+ // This is how the math works out:
+ //
+ // %rsp grows (i.e. gets lower) left to right. Each box below is
+ // one word (eight bytes). Obj0 is the stack slot we're trying to
+ // get to.
+ //
+ // ----------------------------------
+ // | BP | Obj0 | Obj1 | ... | ObjN |
+ // ----------------------------------
+ // ^ ^ ^ ^
+ // A B C E
+ //
+ // A is the incoming stack pointer.
+ // (B - A) is the local area offset (-8 for x86-64) [1]
+ // (C - A) is the Offset returned by MFI->getObjectOffset for Obj0 [2]
+ //
+ // |(E - B)| is the StackSize (absolute value, positive). For a
+ // stack that grown down, this works out to be (B - E). [3]
+ //
+ // E is also the value of %rsp after stack has been set up, and we
+ // want (C - E) -- the value we can add to %rsp to get to Obj0. Now
+ // (C - E) == (C - A) - (B - A) + (B - E)
+ // { Using [1], [2] and [3] above }
+ // == getObjectOffset - LocalAreaOffset + StackSize
+ //
+
+ // Get the Offset from the StackPointer
+ int Offset = MFI->getObjectOffset(FI) - getOffsetOfLocalArea();
+
+ return Offset + StackSize;
+}
+// Simplified from getFrameIndexReference keeping only StackPointer cases
+int X86FrameLowering::getFrameIndexReferenceFromSP(const MachineFunction &MF, int FI,
+ unsigned &FrameReg) const {
+ const X86RegisterInfo *RegInfo =
+ static_cast<const X86RegisterInfo*>(MF.getSubtarget().getRegisterInfo());
+
+ assert(!RegInfo->hasBasePointer(MF) && "we don't handle this case");
+
+ FrameReg = RegInfo->getStackRegister();
+ return getFrameIndexOffsetFromSP(MF, FI);
+}
+
bool X86FrameLowering::assignCalleeSavedSpillSlots(
MachineFunction &MF, const TargetRegisterInfo *TRI,
std::vector<CalleeSavedInfo> &CSI) const {
diff --git a/llvm/lib/Target/X86/X86FrameLowering.h b/llvm/lib/Target/X86/X86FrameLowering.h
index 7740c3ad6f3..2ee71159c19 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.h
+++ b/llvm/lib/Target/X86/X86FrameLowering.h
@@ -69,6 +69,10 @@ public:
int getFrameIndexReference(const MachineFunction &MF, int FI,
unsigned &FrameReg) const override;
+ int getFrameIndexOffsetFromSP(const MachineFunction &MF, int FI) const;
+ int getFrameIndexReferenceFromSP(const MachineFunction &MF, int FI,
+ unsigned &FrameReg) const override;
+
void eliminateCallFramePseudoInstr(MachineFunction &MF,
MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI) const override;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b67bbcb39c3..7faa58cf377 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -21346,6 +21346,11 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
case X86::EH_SjLj_LongJmp64:
return emitEHSjLjLongJmp(MI, BB);
+ case TargetOpcode::STATEPOINT:
+ // As an implementation detail, STATEPOINT shares the STACKMAP format at
+ // this point in the process. We diverge later.
+ return emitPatchPoint(MI, BB);
+
case TargetOpcode::STACKMAP:
case TargetOpcode::PATCHPOINT:
return emitPatchPoint(MI, BB);
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 4e0d594238c..6a10916aa60 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -808,6 +808,66 @@ static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSu
} // while (NumBytes)
}
+static void LowerSTATEPOINT(MCStreamer &OS, StackMaps &SM,
+ const MachineInstr &MI, bool Is64Bit,
+ const TargetMachine& TM,
+ const MCSubtargetInfo& STI,
+ X86MCInstLower &MCInstLowering) {
+ assert(Is64Bit && "Statepoint currently only supports X86-64");
+
+ // We need to record the frame size for stack walking
+ const MachineFunction* MF = MI.getParent()->getParent();
+ assert(MF && "can't find machine function?");
+
+ //
+ // Emit call instruction
+ //
+
+ // Lower call target and choose correct opcode
+ const MachineOperand &call_target = StatepointOpers(&MI).getCallTarget();
+ MCOperand call_target_mcop;
+ unsigned call_opcode;
+ switch (call_target.getType()) {
+ case MachineOperand::MO_GlobalAddress:
+ case MachineOperand::MO_ExternalSymbol:
+ call_target_mcop = MCInstLowering.LowerSymbolOperand(
+ call_target,
+ MCInstLowering.GetSymbolFromOperand(call_target));
+ call_opcode = X86::CALL64pcrel32;
+ // Currently, we only support relative addressing with statepoints.
+ // Otherwise, we'll need a scratch register to hold the target
+ // address. You'll fail asserts during load & relocation if this
+ // symbol is to far away. (TODO: support non-relative addressing)
+ break;
+ case MachineOperand::MO_Immediate:
+ call_target_mcop = MCOperand::CreateImm(call_target.getImm());
+ call_opcode = X86::CALL64pcrel32;
+ // Currently, we only support relative addressing with statepoints.
+ // Otherwise, we'll need a scratch register to hold the target
+ // immediate. You'll fail asserts during load & relocation if this
+ // address is to far away. (TODO: support non-relative addressing)
+ break;
+ case MachineOperand::MO_Register:
+ call_target_mcop = MCOperand::CreateReg(call_target.getReg());
+ call_opcode = X86::CALL64r;
+ break;
+ default:
+ llvm_unreachable("Unsupported operand type in statepoint call target");
+ break;
+ }
+
+ // Emit call
+ MCInst call_inst;
+ call_inst.setOpcode(call_opcode);
+ call_inst.addOperand(call_target_mcop);
+ OS.EmitInstruction(call_inst, STI);
+
+ // Record our statepoint node in the same section used by STACKMAP
+ // and PATCHPOINT
+ SM.recordStatepoint(MI);
+}
+
+
// Lower a stackmap of the form:
// <id>, <shadowBytes>, ...
void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
@@ -1030,7 +1090,9 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
.addExpr(DotExpr));
return;
}
-
+ case TargetOpcode::STATEPOINT:
+ return LowerSTATEPOINT(OutStreamer, SM, *MI, Subtarget->is64Bit(), TM,
+ getSubtargetInfo(), MCInstLowering);
case TargetOpcode::STACKMAP:
return LowerSTACKMAP(*MI);
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp
index 87777d46018..5b301d70edb 100644
--- a/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -300,6 +300,7 @@ void CodeGenTarget::ComputeInstrsByEnum() const {
"IMPLICIT_DEF", "SUBREG_TO_REG", "COPY_TO_REGCLASS", "DBG_VALUE",
"REG_SEQUENCE", "COPY", "BUNDLE", "LIFETIME_START",
"LIFETIME_END", "STACKMAP", "PATCHPOINT", "LOAD_STACK_GUARD",
+ "STATEPOINT",
nullptr};
const DenseMap<const Record*, CodeGenInstruction*> &Insts = getInstructions();
for (const char *const *p = FixedInstrs; *p; ++p) {
OpenPOWER on IntegriCloud