1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
|
//===------- OrcTargetSupport.cpp - Target support utilities for Orc ------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#include "llvm/ADT/Triple.h"
#include "llvm/ExecutionEngine/Orc/OrcTargetSupport.h"
#include "llvm/Support/Process.h"
#include <array>
using namespace llvm::orc;
namespace {
uint64_t executeCompileCallback(JITCompileCallbackManagerBase *JCBM,
TargetAddress CallbackID) {
return JCBM->executeCompileCallback(CallbackID);
}
}
namespace llvm {
namespace orc {
const char* OrcX86_64::ResolverBlockName = "orc_resolver_block";
void OrcX86_64::insertResolverBlock(
Module &M, JITCompileCallbackManagerBase &JCBM) {
// Trampoline code-sequence length, used to get trampoline address from return
// address.
const unsigned X86_64_TrampolineLength = 6;
// List of x86-64 GPRs to save. Note - RBP saved separately below.
std::array<const char *, 14> GPRs = {{
"rax", "rbx", "rcx", "rdx",
"rsi", "rdi", "r8", "r9",
"r10", "r11", "r12", "r13",
"r14", "r15"
}};
// Address of the executeCompileCallback function.
uint64_t CallbackAddr =
static_cast<uint64_t>(
reinterpret_cast<uintptr_t>(executeCompileCallback));
std::ostringstream AsmStream;
Triple TT(M.getTargetTriple());
// Switch to text section.
if (TT.getOS() == Triple::Darwin)
AsmStream << ".section __TEXT,__text,regular,pure_instructions\n"
<< ".align 4, 0x90\n";
else
AsmStream << ".text\n"
<< ".align 16, 0x90\n";
// Bake in a pointer to the callback manager immediately before the
// start of the resolver function.
AsmStream << "jit_callback_manager_addr:\n"
<< " .quad " << &JCBM << "\n";
// Start the resolver function.
AsmStream << ResolverBlockName << ":\n"
<< " pushq %rbp\n"
<< " movq %rsp, %rbp\n";
// Store the GPRs.
for (const auto &GPR : GPRs)
AsmStream << " pushq %" << GPR << "\n";
// Store floating-point state with FXSAVE.
// Note: We need to keep the stack 16-byte aligned, so if we've emitted an odd
// number of 64-bit pushes so far (GPRs.size() plus 1 for RBP) then add
// an extra 64 bits of padding to the FXSave area.
unsigned Padding = (GPRs.size() + 1) % 2 ? 8 : 0;
unsigned FXSaveSize = 512 + Padding;
AsmStream << " subq $" << FXSaveSize << ", %rsp\n"
<< " fxsave64 (%rsp)\n"
// Load callback manager address, compute trampoline address, call JIT.
<< " lea jit_callback_manager_addr(%rip), %rdi\n"
<< " movq (%rdi), %rdi\n"
<< " movq 0x8(%rbp), %rsi\n"
<< " subq $" << X86_64_TrampolineLength << ", %rsi\n"
<< " movabsq $" << CallbackAddr << ", %rax\n"
<< " callq *%rax\n"
// Replace the return to the trampoline with the return address of the
// compiled function body.
<< " movq %rax, 0x8(%rbp)\n"
// Restore the floating point state.
<< " fxrstor64 (%rsp)\n"
<< " addq $" << FXSaveSize << ", %rsp\n";
for (const auto &GPR : make_range(GPRs.rbegin(), GPRs.rend()))
AsmStream << " popq %" << GPR << "\n";
// Restore original RBP and return to compiled function body.
AsmStream << " popq %rbp\n"
<< " retq\n";
M.appendModuleInlineAsm(AsmStream.str());
}
OrcX86_64::LabelNameFtor
OrcX86_64::insertCompileCallbackTrampolines(Module &M,
TargetAddress ResolverBlockAddr,
unsigned NumCalls,
unsigned StartIndex) {
const char *ResolverBlockPtrName = "Lorc_resolve_block_addr";
std::ostringstream AsmStream;
Triple TT(M.getTargetTriple());
if (TT.getOS() == Triple::Darwin)
AsmStream << ".section __TEXT,__text,regular,pure_instructions\n"
<< ".align 4, 0x90\n";
else
AsmStream << ".text\n"
<< ".align 16, 0x90\n";
AsmStream << ResolverBlockPtrName << ":\n"
<< " .quad " << ResolverBlockAddr << "\n";
auto GetLabelName =
[=](unsigned I) {
std::ostringstream LabelStream;
LabelStream << "orc_jcc_" << (StartIndex + I);
return LabelStream.str();
};
for (unsigned I = 0; I < NumCalls; ++I)
AsmStream << GetLabelName(I) << ":\n"
<< " callq *" << ResolverBlockPtrName << "(%rip)\n";
M.appendModuleInlineAsm(AsmStream.str());
return GetLabelName;
}
OrcX86_64::IndirectStubsInfo::IndirectStubsInfo(IndirectStubsInfo &&Other) {
StubsBlock = std::move(Other.StubsBlock);
PtrsBlock = std::move(Other.PtrsBlock);
Other.StubsBlock = sys::MemoryBlock();
Other.PtrsBlock = sys::MemoryBlock();
}
OrcX86_64::IndirectStubsInfo&
OrcX86_64::IndirectStubsInfo::operator=(IndirectStubsInfo &&Other) {
StubsBlock = std::move(Other.StubsBlock);
PtrsBlock = std::move(Other.PtrsBlock);
Other.StubsBlock = sys::MemoryBlock();
Other.PtrsBlock = sys::MemoryBlock();
return *this;
}
OrcX86_64::IndirectStubsInfo::~IndirectStubsInfo() {
sys::Memory::releaseMappedMemory(StubsBlock);
sys::Memory::releaseMappedMemory(PtrsBlock);
}
std::error_code OrcX86_64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo,
unsigned MinStubs,
void *InitialPtrVal) {
// Stub format is:
//
// .section __orc_stubs
// stub1:
// jmpq *ptr1(%rip)
// .byte 0xC4 ; <- Invalid opcode padding.
// .byte 0xF1
// stub2:
// jmpq *ptr2(%rip)
//
// ...
//
// .section __orc_ptrs
// ptr1:
// .quad 0x0
// ptr2:
// .quad 0x0
//
// ...
const unsigned StubSize = IndirectStubsInfo::StubSize;
// Emit at least MinStubs, rounded up to fill the pages allocated.
unsigned PageSize = sys::Process::getPageSize();
unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize;
unsigned NumStubs = (NumPages * PageSize) / StubSize;
// Allocate memory for stubs and pointers in one call.
std::error_code EC;
auto InitialBlock = sys::Memory::allocateMappedMemory(2 * NumPages * PageSize,
nullptr,
sys::Memory::MF_READ |
sys::Memory::MF_WRITE,
EC);
if (EC)
return EC;
// Create separate MemoryBlocks representing the stubs and pointers.
sys::MemoryBlock StubsBlock(InitialBlock.base(), NumPages * PageSize);
sys::MemoryBlock PtrsBlock(static_cast<char*>(InitialBlock.base()) +
NumPages * PageSize,
NumPages * PageSize);
// Populate the stubs page stubs and mark it executable.
uint64_t *Stub = reinterpret_cast<uint64_t*>(StubsBlock.base());
uint64_t PtrOffsetField =
static_cast<uint64_t>(NumPages * PageSize - 6) << 16;
for (unsigned I = 0; I < NumStubs; ++I)
Stub[I] = 0xF1C40000000025ff | PtrOffsetField;
if (auto EC = sys::Memory::protectMappedMemory(StubsBlock,
sys::Memory::MF_READ |
sys::Memory::MF_EXEC))
return EC;
// Initialize all pointers to point at FailureAddress.
void **Ptr = reinterpret_cast<void**>(PtrsBlock.base());
for (unsigned I = 0; I < NumStubs; ++I)
Ptr[I] = InitialPtrVal;
StubsInfo.NumStubs = NumStubs;
StubsInfo.StubsBlock = std::move(StubsBlock);
StubsInfo.PtrsBlock = std::move(PtrsBlock);
return std::error_code();
}
} // End namespace orc.
} // End namespace llvm.
|
