summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CodeGen/CMakeLists.txt1
-rw-r--r--llvm/lib/CodeGen/CodeGen.cpp1
-rw-r--r--llvm/lib/CodeGen/Passes.cpp2
-rw-r--r--llvm/lib/CodeGen/PatchableFunction.cpp70
-rw-r--r--llvm/lib/Target/X86/X86AsmPrinter.cpp4
-rw-r--r--llvm/lib/Target/X86/X86AsmPrinter.h12
-rw-r--r--llvm/lib/Target/X86/X86MCInstLower.cpp65
7 files changed, 137 insertions, 18 deletions
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 596a2d9907e..cb646d5717f 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -76,6 +76,7 @@ add_llvm_library(LLVMCodeGen
MachineSSAUpdater.cpp
MachineTraceMetrics.cpp
MachineVerifier.cpp
+ PatchableFunction.cpp
MIRPrinter.cpp
MIRPrintingPass.cpp
OptimizePHIs.cpp
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 3c2dc85df59..2613f5e4eda 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -55,6 +55,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
initializeMachineSchedulerPass(Registry);
initializeMachineSinkingPass(Registry);
initializeMachineVerifierPassPass(Registry);
+ initializePatchableFunctionPass(Registry);
initializeOptimizePHIsPass(Registry);
initializePEIPass(Registry);
initializePHIEliminationPass(Registry);
diff --git a/llvm/lib/CodeGen/Passes.cpp b/llvm/lib/CodeGen/Passes.cpp
index fa5c58df0c0..94b42089060 100644
--- a/llvm/lib/CodeGen/Passes.cpp
+++ b/llvm/lib/CodeGen/Passes.cpp
@@ -602,6 +602,8 @@ void TargetPassConfig::addMachinePasses() {
addPass(&StackMapLivenessID, false);
addPass(&LiveDebugValuesID, false);
+ addPass(&PatchableFunctionID, false);
+
AddingMachinePasses = false;
}
diff --git a/llvm/lib/CodeGen/PatchableFunction.cpp b/llvm/lib/CodeGen/PatchableFunction.cpp
new file mode 100644
index 00000000000..ca345dfaf2b
--- /dev/null
+++ b/llvm/lib/CodeGen/PatchableFunction.cpp
@@ -0,0 +1,70 @@
+//===-- PatchableFunction.cpp - Patchable prologues for LLVM -------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements edits function bodies in place to support the
+// "patchable-function" attribute.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/Analysis.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+
+using namespace llvm;
+
+namespace {
+struct PatchableFunction : public MachineFunctionPass {
+ static char ID; // Pass identification, replacement for typeid
+ PatchableFunction() : MachineFunctionPass(ID) {
+ initializePatchableFunctionPass(*PassRegistry::getPassRegistry());
+ }
+
+ bool runOnMachineFunction(MachineFunction &F) override;
+ MachineFunctionProperties getRequiredProperties() const override {
+ return MachineFunctionProperties().set(
+ MachineFunctionProperties::Property::AllVRegsAllocated);
+ }
+};
+}
+
+bool PatchableFunction::runOnMachineFunction(MachineFunction &MF) {
+ if (!MF.getFunction()->hasFnAttribute("patchable-function"))
+ return false;
+
+#ifndef NDEBUG
+ Attribute PatchAttr = MF.getFunction()->getFnAttribute("patchable-function");
+ StringRef PatchType = PatchAttr.getValueAsString();
+ assert(PatchType == "prologue-short-redirect" && "Only possibility today!");
+#endif
+
+ auto &FirstMBB = *MF.begin();
+ auto &FirstMI = *FirstMBB.begin();
+
+ auto *TII = MF.getSubtarget().getInstrInfo();
+ auto MIB = BuildMI(FirstMBB, FirstMBB.begin(), FirstMI.getDebugLoc(),
+ TII->get(TargetOpcode::PATCHABLE_OP))
+ .addImm(2)
+ .addImm(FirstMI.getOpcode());
+
+ for (auto &MO : FirstMI.operands())
+ MIB.addOperand(MO);
+
+ FirstMI.eraseFromParent();
+ MF.ensureAlignment(4);
+ return true;
+}
+
+char PatchableFunction::ID = 0;
+char &llvm::PatchableFunctionID = PatchableFunction::ID;
+INITIALIZE_PASS(PatchableFunction, "patchable-function", "", false, false)
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 9abe91b0bc0..9d0ac319859 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -27,6 +27,7 @@
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSectionCOFF.h"
@@ -49,6 +50,9 @@ bool X86AsmPrinter::runOnMachineFunction(MachineFunction &MF) {
Subtarget = &MF.getSubtarget<X86Subtarget>();
SMShadowTracker.startFunction(MF);
+ CodeEmitter.reset(TM.getTarget().createMCCodeEmitter(
+ *MF.getSubtarget().getInstrInfo(), *MF.getSubtarget().getRegisterInfo(),
+ MF.getContext()));
SetupMachineFunction(MF);
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h
index 9c8bd98dbad..0abb4f32965 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.h
+++ b/llvm/lib/Target/X86/X86AsmPrinter.h
@@ -29,6 +29,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
const X86Subtarget *Subtarget;
StackMaps SM;
FaultMaps FM;
+ std::unique_ptr<MCCodeEmitter> CodeEmitter;
// This utility class tracks the length of a stackmap instruction's 'shadow'.
// It is used by the X86AsmPrinter to ensure that the stackmap shadow
@@ -40,10 +41,11 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
// few instruction bytes to cover the shadow are NOPs used for padding.
class StackMapShadowTracker {
public:
- StackMapShadowTracker(TargetMachine &TM);
+ StackMapShadowTracker();
~StackMapShadowTracker();
void startFunction(MachineFunction &MF);
- void count(MCInst &Inst, const MCSubtargetInfo &STI);
+ void count(MCInst &Inst, const MCSubtargetInfo &STI,
+ MCCodeEmitter *CodeEmitter);
// Called to signal the start of a shadow of RequiredSize bytes.
void reset(unsigned RequiredSize) {
@@ -56,9 +58,7 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
// to emit any necessary padding-NOPs.
void emitShadowPadding(MCStreamer &OutStreamer, const MCSubtargetInfo &STI);
private:
- TargetMachine &TM;
const MachineFunction *MF;
- std::unique_ptr<MCCodeEmitter> CodeEmitter;
bool InShadow;
// RequiredShadowSize holds the length of the shadow specified in the most
@@ -82,14 +82,14 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
void LowerPATCHPOINT(const MachineInstr &MI, X86MCInstLower &MCIL);
void LowerSTATEPOINT(const MachineInstr &MI, X86MCInstLower &MCIL);
void LowerFAULTING_LOAD_OP(const MachineInstr &MI, X86MCInstLower &MCIL);
+ void LowerPATCHABLE_OP(const MachineInstr &MI, X86MCInstLower &MCIL);
void LowerTlsAddr(X86MCInstLower &MCInstLowering, const MachineInstr &MI);
public:
explicit X86AsmPrinter(TargetMachine &TM,
std::unique_ptr<MCStreamer> Streamer)
- : AsmPrinter(TM, std::move(Streamer)), SM(*this), FM(*this),
- SMShadowTracker(TM) {}
+ : AsmPrinter(TM, std::move(Streamer)), SM(*this), FM(*this) {}
const char *getPassName() const override {
return "X86 Assembly / Object Emitter";
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index d02786cf562..ce3df749fcc 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -20,6 +20,7 @@
#include "Utils/X86ShuffleDecode.h"
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/iterator_range.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineOperand.h"
@@ -70,24 +71,21 @@ private:
// Emit a minimal sequence of nops spanning NumBytes bytes.
static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit,
- const MCSubtargetInfo &STI);
+ const MCSubtargetInfo &STI, bool OnlyOneNop = false);
namespace llvm {
- X86AsmPrinter::StackMapShadowTracker::StackMapShadowTracker(TargetMachine &TM)
- : TM(TM), InShadow(false), RequiredShadowSize(0), CurrentShadowSize(0) {}
+ X86AsmPrinter::StackMapShadowTracker::StackMapShadowTracker()
+ : InShadow(false), RequiredShadowSize(0), CurrentShadowSize(0) {}
X86AsmPrinter::StackMapShadowTracker::~StackMapShadowTracker() {}
- void
- X86AsmPrinter::StackMapShadowTracker::startFunction(MachineFunction &F) {
+ void X86AsmPrinter::StackMapShadowTracker::startFunction(MachineFunction &F) {
MF = &F;
- CodeEmitter.reset(TM.getTarget().createMCCodeEmitter(
- *MF->getSubtarget().getInstrInfo(),
- *MF->getSubtarget().getRegisterInfo(), MF->getContext()));
}
void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst,
- const MCSubtargetInfo &STI) {
+ const MCSubtargetInfo &STI,
+ MCCodeEmitter *CodeEmitter) {
if (InShadow) {
SmallString<256> Code;
SmallVector<MCFixup, 4> Fixups;
@@ -110,7 +108,7 @@ namespace llvm {
void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
OutStreamer->EmitInstruction(Inst, getSubtargetInfo());
- SMShadowTracker.count(Inst, getSubtargetInfo());
+ SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get());
}
} // end llvm namespace
@@ -786,7 +784,8 @@ void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
}
/// \brief Emit the optimal amount of multi-byte nops on X86.
-static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSubtargetInfo &STI) {
+static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit,
+ const MCSubtargetInfo &STI, bool OnlyOneNop) {
// This works only for 64bit. For 32bit we have to do additional checking if
// the CPU supports multi-byte nops.
assert(Is64Bit && "EmitNops only supports X86-64");
@@ -833,6 +832,10 @@ static void EmitNops(MCStreamer &OS, unsigned NumBytes, bool Is64Bit, const MCSu
.addImm(Displacement).addReg(SegmentReg), STI);
break;
}
+
+ (void) OnlyOneNop;
+ assert((!OnlyOneNop || NumBytes == 0) &&
+ "Allowed only one nop instruction!");
} // while (NumBytes)
}
@@ -915,6 +918,41 @@ void X86AsmPrinter::LowerFAULTING_LOAD_OP(const MachineInstr &MI,
OutStreamer->EmitInstruction(LoadMI, getSubtargetInfo());
}
+void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
+ X86MCInstLower &MCIL) {
+ // PATCHABLE_OP minsize, opcode, operands
+
+ unsigned MinSize = MI.getOperand(0).getImm();
+ unsigned Opcode = MI.getOperand(1).getImm();
+
+ MCInst MCI;
+ MCI.setOpcode(Opcode);
+ for (auto &MO : make_range(MI.operands_begin() + 2, MI.operands_end()))
+ if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
+ MCI.addOperand(MaybeOperand.getValue());
+
+ SmallString<256> Code;
+ SmallVector<MCFixup, 4> Fixups;
+ raw_svector_ostream VecOS(Code);
+ CodeEmitter->encodeInstruction(MCI, VecOS, Fixups, getSubtargetInfo());
+
+ if (Code.size() < MinSize) {
+ if (MinSize == 2 && Opcode == X86::PUSH64r) {
+ // This is an optimization that lets us get away without emitting a nop in
+ // many cases.
+ //
+ // NB! In some cases the encoding for PUSH64r (e.g. PUSH64r %R9) takes two
+ // bytes too, so the check on MinSize is important.
+ MCI.setOpcode(X86::PUSH64rmr);
+ } else {
+ EmitNops(*OutStreamer, MinSize, Subtarget->is64Bit(), getSubtargetInfo(),
+ /* OnlyOneNop = */ true);
+ }
+ }
+
+ OutStreamer->EmitInstruction(MCI, getSubtargetInfo());
+}
+
// Lower a stackmap of the form:
// <id>, <shadowBytes>, ...
void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
@@ -1213,6 +1251,9 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
case TargetOpcode::FAULTING_LOAD_OP:
return LowerFAULTING_LOAD_OP(*MI, MCInstLowering);
+ case TargetOpcode::PATCHABLE_OP:
+ return LowerPATCHABLE_OP(*MI, MCInstLowering);
+
case TargetOpcode::STACKMAP:
return LowerSTACKMAP(*MI);
@@ -1475,7 +1516,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
// is at the end of the shadow.
if (MI->isCall()) {
// Count then size of the call towards the shadow
- SMShadowTracker.count(TmpInst, getSubtargetInfo());
+ SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get());
// Then flush the shadow so that we fill with nops before the call, not
// after it.
SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
OpenPOWER on IntegriCloud