summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp10
-rw-r--r--llvm/lib/CodeGen/XRayInstrumentation.cpp110
-rw-r--r--llvm/lib/Target/ARM/ARMAsmPrinter.cpp9
-rw-r--r--llvm/lib/Target/ARM/ARMAsmPrinter.h12
-rw-r--r--llvm/lib/Target/ARM/ARMBaseInstrInfo.h4
-rw-r--r--llvm/lib/Target/ARM/ARMMCInstLower.cpp87
-rw-r--r--llvm/lib/Target/ARM/ARMSubtarget.cpp5
-rw-r--r--llvm/lib/Target/ARM/ARMSubtarget.h2
-rw-r--r--llvm/lib/Target/X86/X86AsmPrinter.h23
-rw-r--r--llvm/lib/Target/X86/X86MCInstLower.cpp10
-rw-r--r--llvm/lib/Target/X86/X86Subtarget.h2
11 files changed, 213 insertions, 61 deletions
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 0fed4e91eef..ccc1691232c 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2606,3 +2606,13 @@ GCMetadataPrinter *AsmPrinter::GetOrCreateGCPrinter(GCStrategy &S) {
AsmPrinterHandler::~AsmPrinterHandler() {}
void AsmPrinterHandler::markFunctionEnd() {}
+
+void AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI,
+ SledKind Kind) {
+ auto Fn = MI.getParent()->getParent()->getFunction();
+ auto Attr = Fn->getFnAttribute("function-instrument");
+ bool AlwaysInstrument =
+ Attr.isStringAttribute() && Attr.getValueAsString() == "xray-always";
+ Sleds.emplace_back(
+ XRayFunctionEntry{ Sled, CurrentFnSym, Kind, AlwaysInstrument, Fn });
+}
diff --git a/llvm/lib/CodeGen/XRayInstrumentation.cpp b/llvm/lib/CodeGen/XRayInstrumentation.cpp
index 043a1a48db6..ea29d4cb597 100644
--- a/llvm/lib/CodeGen/XRayInstrumentation.cpp
+++ b/llvm/lib/CodeGen/XRayInstrumentation.cpp
@@ -34,7 +34,74 @@ struct XRayInstrumentation : public MachineFunctionPass {
}
bool runOnMachineFunction(MachineFunction &MF) override;
+
+private:
+ // Replace the original RET instruction with the exit sled code ("patchable
+ // ret" pseudo-instruction), so that at runtime XRay can replace the sled
+ // with a code jumping to XRay trampoline, which calls the tracing handler
+ // and, in the end, issues the RET instruction.
+ // This is the approach to go on CPUs which have a single RET instruction,
+ // like x86/x86_64.
+ void replaceRetWithPatchableRet(MachineFunction &MF,
+ const TargetInstrInfo *TII);
+ // Prepend the original return instruction with the exit sled code ("patchable
+ // function exit" pseudo-instruction), preserving the original return
+ // instruction just after the exit sled code.
+ // This is the approach to go on CPUs which have multiple options for the
+ // return instruction, like ARM. For such CPUs we can't just jump into the
+ // XRay trampoline and issue a single return instruction there. We rather
+ // have to call the trampoline and return from it to the original return
+ // instruction of the function being instrumented.
+ void prependRetWithPatchableExit(MachineFunction &MF,
+ const TargetInstrInfo *TII);
};
+} // anonymous namespace
+
+void XRayInstrumentation::replaceRetWithPatchableRet(MachineFunction &MF,
+ const TargetInstrInfo *TII)
+{
+ // We look for *all* terminators and returns, then replace those with
+ // PATCHABLE_RET instructions.
+ SmallVector<MachineInstr *, 4> Terminators;
+ for (auto &MBB : MF) {
+ for (auto &T : MBB.terminators()) {
+ unsigned Opc = 0;
+ if (T.isReturn() && T.getOpcode() == TII->getReturnOpcode()) {
+ // Replace return instructions with:
+ // PATCHABLE_RET <Opcode>, <Operand>...
+ Opc = TargetOpcode::PATCHABLE_RET;
+ }
+ if (TII->isTailCall(T)) {
+ // Treat the tail call as a return instruction, which has a
+ // different-looking sled than the normal return case.
+ Opc = TargetOpcode::PATCHABLE_TAIL_CALL;
+ }
+ if (Opc != 0) {
+ auto MIB = BuildMI(MBB, T, T.getDebugLoc(), TII->get(Opc))
+ .addImm(T.getOpcode());
+ for (auto &MO : T.operands())
+ MIB.addOperand(MO);
+ Terminators.push_back(&T);
+ }
+ }
+ }
+
+ for (auto &I : Terminators)
+ I->eraseFromParent();
+}
+
+void XRayInstrumentation::prependRetWithPatchableExit(MachineFunction &MF,
+ const TargetInstrInfo *TII)
+{
+ for (auto &MBB : MF) {
+ for (auto &T : MBB.terminators()) {
+ if (T.isReturn()) {
+ // Prepend the return instruction with PATCHABLE_FUNCTION_EXIT
+ auto MIB = BuildMI(MBB, T, T.getDebugLoc(),
+ TII->get(TargetOpcode::PATCHABLE_FUNCTION_EXIT));
+ }
+ }
+ }
}
bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
@@ -54,6 +121,11 @@ bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
return false; // Function is too small.
}
+ if (!MF.getSubtarget().isXRaySupported()) {
+ //FIXME: can this be reported somehow?
+ return false;
+ }
+
// FIXME: Do the loop triviality analysis here or in an earlier pass.
// First, insert an PATCHABLE_FUNCTION_ENTER as the first instruction of the
@@ -64,35 +136,17 @@ bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
BuildMI(FirstMBB, FirstMI, FirstMI.getDebugLoc(),
TII->get(TargetOpcode::PATCHABLE_FUNCTION_ENTER));
- // Then we look for *all* terminators and returns, then replace those with
- // PATCHABLE_RET instructions.
- SmallVector<MachineInstr *, 4> Terminators;
- for (auto &MBB : MF) {
- for (auto &T : MBB.terminators()) {
- unsigned Opc = 0;
- if (T.isReturn() && T.getOpcode() == TII->getReturnOpcode()) {
- // Replace return instructions with:
- // PATCHABLE_RET <Opcode>, <Operand>...
- Opc = TargetOpcode::PATCHABLE_RET;
- }
- if (TII->isTailCall(T)) {
- // Treat the tail call as a return instruction, which has a
- // different-looking sled than the normal return case.
- Opc = TargetOpcode::PATCHABLE_TAIL_CALL;
- }
- if (Opc != 0) {
- auto MIB = BuildMI(MBB, T, T.getDebugLoc(), TII->get(Opc))
- .addImm(T.getOpcode());
- for (auto &MO : T.operands())
- MIB.addOperand(MO);
- Terminators.push_back(&T);
- }
- }
+ switch (MF.getTarget().getTargetTriple().getArch()) {
+ case Triple::ArchType::arm:
+ // For the architectures which don't have a single return instruction
+ prependRetWithPatchableExit(MF, TII);
+ break;
+ default:
+ // For the architectures that have a single return instruction (such as
+ // RETQ on x86_64).
+ replaceRetWithPatchableRet(MF, TII);
+ break;
}
-
- for (auto &I : Terminators)
- I->eraseFromParent();
-
return true;
}
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index d19a2b9466f..2d8e7256207 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -150,6 +150,9 @@ bool ARMAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
// Emit the rest of the function body.
EmitFunctionBody();
+ // Emit the XRay table for this function.
+ EmitXRayTable();
+
// If we need V4T thumb mode Register Indirect Jump pads, emit them.
// These are created per function, rather than per TU, since it's
// relatively easy to exceed the thumb branch range within a TU.
@@ -2005,6 +2008,12 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
.addReg(0));
return;
}
+ case ARM::PATCHABLE_FUNCTION_ENTER:
+ LowerPATCHABLE_FUNCTION_ENTER(*MI);
+ return;
+ case ARM::PATCHABLE_FUNCTION_EXIT:
+ LowerPATCHABLE_FUNCTION_EXIT(*MI);
+ return;
}
MCInst TmpInst;
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.h b/llvm/lib/Target/ARM/ARMAsmPrinter.h
index 97f5ca0ecbc..fe4a2bfca35 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.h
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.h
@@ -94,7 +94,19 @@ public:
// lowerOperand - Convert a MachineOperand into the equivalent MCOperand.
bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp);
+ //===------------------------------------------------------------------===//
+ // XRay implementation
+ //===------------------------------------------------------------------===//
+public:
+ // XRay-specific lowering for ARM.
+ void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
+ void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
+ // Helper function that emits the XRay sleds we've collected for a particular
+ // function.
+ void EmitXRayTable();
+
private:
+ void EmitSled(const MachineInstr &MI, SledKind Kind);
// Helpers for EmitStartOfAsmFile() and EmitEndOfAsmFile()
void emitAttributes();
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
index cbc4a02bb3c..e8b3720522b 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -100,6 +100,10 @@ public:
// Return whether the target has an explicit NOP encoding.
bool hasNOP() const;
+ virtual void getNoopForElfTarget(MCInst &NopInst) const {
+ getNoopForMachoTarget(NopInst);
+ }
+
// Return the non-pre/post incrementing version of 'Opc'. Return 0
// if there is not such an opcode.
virtual unsigned getUnindexedOpcode(unsigned Opc) const =0;
diff --git a/llvm/lib/Target/ARM/ARMMCInstLower.cpp b/llvm/lib/Target/ARM/ARMMCInstLower.cpp
index 7429acdb09a..afdde4e11db 100644
--- a/llvm/lib/Target/ARM/ARMMCInstLower.cpp
+++ b/llvm/lib/Target/ARM/ARMMCInstLower.cpp
@@ -21,6 +21,11 @@
#include "llvm/IR/Mangler.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSymbolELF.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCInstBuilder.h"
+#include "llvm/MC/MCStreamer.h"
using namespace llvm;
@@ -150,3 +155,85 @@ void llvm::LowerARMMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
}
}
}
+
+void ARMAsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
+{
+ static const int8_t NoopsInSledCount = 6;
+ // We want to emit the following pattern:
+ //
+ // .Lxray_sled_N:
+ // ALIGN
+ // B #20
+ // ; 6 NOP instructions (24 bytes)
+ // .tmpN
+ //
+ // We need the 24 bytes (6 instructions) because at runtime, we'd be patching
+ // over the full 28 bytes (7 instructions) with the following pattern:
+ //
+ // PUSH{ r0, lr }
+ // MOVW r0, #<lower 16 bits of function ID>
+ // MOVT r0, #<higher 16 bits of function ID>
+ // MOVW ip, #<lower 16 bits of address of __xray_FunctionEntry/Exit>
+ // MOVT ip, #<higher 16 bits of address of __xray_FunctionEntry/Exit>
+ // BLX ip
+ // POP{ r0, lr }
+ //
+ OutStreamer->EmitCodeAlignment(4);
+ auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
+ OutStreamer->EmitLabel(CurSled);
+ auto Target = OutContext.createTempSymbol();
+
+ // Emit "B #20" instruction, which jumps over the next 24 bytes (because
+ // register pc is 8 bytes ahead of the jump instruction by the moment CPU
+ // is executing it).
+ // By analogy to ARMAsmPrinter::emitPseudoExpansionLowering() |case ARM::B|.
+ // It is not clear why |addReg(0)| is needed (the last operand).
+ EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::Bcc).addImm(20)
+ .addImm(ARMCC::AL).addReg(0));
+
+ MCInst Noop;
+ Subtarget->getInstrInfo()->getNoopForElfTarget(Noop);
+ for (int8_t I = 0; I < NoopsInSledCount; I++)
+ {
+ OutStreamer->EmitInstruction(Noop, getSubtargetInfo());
+ }
+
+ OutStreamer->EmitLabel(Target);
+ recordSled(CurSled, MI, Kind);
+}
+
+void ARMAsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
+{
+ EmitSled(MI, SledKind::FUNCTION_ENTER);
+}
+
+void ARMAsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
+{
+ EmitSled(MI, SledKind::FUNCTION_EXIT);
+}
+
+void ARMAsmPrinter::EmitXRayTable()
+{
+ if (Sleds.empty())
+ return;
+ if (Subtarget->isTargetELF()) {
+ auto *Section = OutContext.getELFSection(
+ "xray_instr_map", ELF::SHT_PROGBITS,
+ ELF::SHF_ALLOC | ELF::SHF_GROUP | ELF::SHF_MERGE, 0,
+ CurrentFnSym->getName());
+ auto PrevSection = OutStreamer->getCurrentSectionOnly();
+ OutStreamer->SwitchSection(Section);
+ for (const auto &Sled : Sleds) {
+ OutStreamer->EmitSymbolValue(Sled.Sled, 4);
+ OutStreamer->EmitSymbolValue(CurrentFnSym, 4);
+ auto Kind = static_cast<uint8_t>(Sled.Kind);
+ OutStreamer->EmitBytes(
+ StringRef(reinterpret_cast<const char *>(&Kind), 1));
+ OutStreamer->EmitBytes(
+ StringRef(reinterpret_cast<const char *>(&Sled.AlwaysInstrument), 1));
+ OutStreamer->EmitZeros(6);
+ }
+ OutStreamer->SwitchSection(PrevSection);
+ }
+ Sleds.clear();
+}
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 469d340d122..481aacdc5b1 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -101,6 +101,11 @@ ARMSubtarget::ARMSubtarget(const Triple &TT, const std::string &CPU,
: (ARMBaseInstrInfo *)new Thumb2InstrInfo(*this)),
TLInfo(TM, *this) {}
+bool ARMSubtarget::isXRaySupported() const {
+ // We don't currently suppport Thumb, but Windows requires Thumb.
+ return hasV6Ops() && !isTargetWindows();
+}
+
void ARMSubtarget::initializeEnvironment() {
// MCAsmInfo isn't always present (e.g. in opt) so we can't initialize this
// directly from it, but we can try to make sure they're consistent when both
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 2c14eb094b5..6d528b8d62f 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -540,6 +540,8 @@ public:
}
bool isTargetAndroid() const { return TargetTriple.isAndroid(); }
+ virtual bool isXRaySupported() const override;
+
bool isAPCS_ABI() const;
bool isAAPCS_ABI() const;
bool isAAPCS16_ABI() const;
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.h b/llvm/lib/Target/X86/X86AsmPrinter.h
index dcb7b5a3466..50acb391f54 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.h
+++ b/llvm/lib/Target/X86/X86AsmPrinter.h
@@ -71,27 +71,6 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
StackMapShadowTracker SMShadowTracker;
- // This describes the kind of sled we're storing in the XRay table.
- enum class SledKind : uint8_t {
- FUNCTION_ENTER = 0,
- FUNCTION_EXIT = 1,
- TAIL_CALL = 2,
- };
-
- // The table will contain these structs that point to the sled, the function
- // containing the sled, and what kind of sled (and whether they should always
- // be instrumented).
- struct XRayFunctionEntry {
- const MCSymbol *Sled;
- const MCSymbol *Function;
- SledKind Kind;
- bool AlwaysInstrument;
- const class Function *Fn;
- };
-
- // All the sleds to be emitted.
- std::vector<XRayFunctionEntry> Sleds;
-
// All instructions emitted by the X86AsmPrinter should use this helper
// method.
//
@@ -117,8 +96,6 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
// function.
void EmitXRayTable();
- // Helper function to record a given XRay sled.
- void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind);
public:
explicit X86AsmPrinter(TargetMachine &TM,
std::unique_ptr<MCStreamer> Streamer)
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 46197756566..b165360b66d 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -1022,16 +1022,6 @@ void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
getSubtargetInfo());
}
-void X86AsmPrinter::recordSled(MCSymbol *Sled, const MachineInstr &MI,
- SledKind Kind) {
- auto Fn = MI.getParent()->getParent()->getFunction();
- auto Attr = Fn->getFnAttribute("function-instrument");
- bool AlwaysInstrument =
- Attr.isStringAttribute() && Attr.getValueAsString() == "xray-always";
- Sleds.emplace_back(
- XRayFunctionEntry{Sled, CurrentFnSym, Kind, AlwaysInstrument, Fn});
-}
-
void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
X86MCInstLower &MCIL) {
// We want to emit the following pattern:
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index c1f862d5077..a5cd83d2514 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -460,6 +460,8 @@ public:
bool hasPKU() const { return HasPKU; }
bool hasMPX() const { return HasMPX; }
+ virtual bool isXRaySupported() const override { return is64Bit(); }
+
bool isAtom() const { return X86ProcFamily == IntelAtom; }
bool isSLM() const { return X86ProcFamily == IntelSLM; }
bool useSoftFloat() const { return UseSoftFloat; }
OpenPOWER on IntegriCloud