diff options
| -rw-r--r-- | llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h | 40 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/SectionMemoryManager.cpp | 89 | ||||
| -rw-r--r-- | llvm/lib/Support/Unix/Memory.inc | 5 |
3 files changed, 90 insertions, 44 deletions
diff --git a/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h b/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h index 6135bc68e9a..7bb96eb8b71 100644 --- a/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h +++ b/llvm/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -83,25 +83,28 @@ public: virtual void invalidateInstructionCache(); private: + struct FreeMemBlock { + // The actual block of free memory + sys::MemoryBlock Free; + // If there is a pending allocation from the same reservation right before + // this block, store it's index in PendingMem, to be able to update the + // pending region if part of this block is allocated, rather than having to + // create a new one + unsigned PendingPrefixIndex; + }; + struct MemoryGroup { - // PendingMem contains all allocated memory blocks - // which have not yet had their permissions set. Note - // that this tracks memory blocks that have been given to - // this memory manager by the system, not those - // given out to the user. In particular, the memory manager - // will give out subblocks of these MemoryBlocks in response - // to user requests. We track which subblocks have not beeen - // given out yet in `FreeMem`. - SmallVector<sys::MemoryBlock, 16> PendingMem; - SmallVector<sys::MemoryBlock, 16> FreeMem; - - // All allocated memory blocks that have had their permissions - // set (i.e. that have been finalized). Because of this, we may - // not give out subblocks of this memory to the user anymore, - // even if those subblocks have not been previously given out. - SmallVector<sys::MemoryBlock, 16> AllocatedMem; - - sys::MemoryBlock Near; + // PendingMem contains all blocks of memory (subblocks of AllocatedMem) + // which have not yet had their permissions applied, but have been given + // out to the user. FreeMem contains all block of memory, which have + // neither had their permissions applied, nor been given out to the user. + SmallVector<sys::MemoryBlock, 16> PendingMem; + SmallVector<FreeMemBlock, 16> FreeMem; + + // All memory blocks that have been requested from the system + SmallVector<sys::MemoryBlock, 16> AllocatedMem; + + sys::MemoryBlock Near; }; uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size, @@ -118,4 +121,3 @@ private: } #endif // LLVM_EXECUTION_ENGINE_SECTION_MEMORY_MANAGER_H - diff --git a/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp b/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp index 5e89a945b80..e2f220862cf 100644 --- a/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -15,6 +15,7 @@ #include "llvm/Config/config.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/Process.h" namespace llvm { @@ -48,14 +49,27 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // Look in the list of free memory regions and use a block there if one // is available. - for (sys::MemoryBlock &MB : MemGroup.FreeMem) { - if (MB.size() >= RequiredSize) { - Addr = (uintptr_t)MB.base(); - uintptr_t EndOfBlock = Addr + MB.size(); + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.size() >= RequiredSize) { + Addr = (uintptr_t)FreeMB.Free.base(); + uintptr_t EndOfBlock = Addr + FreeMB.Free.size(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); - // Store cutted free memory block. - MB = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); + + if (FreeMB.PendingPrefixIndex == (unsigned)-1) { + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + + // Remember this pending block, such that future allocations can just + // modify it rather than creating a new one + FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; + } else { + sys::MemoryBlock &PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; + PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base()); + } + + // Remember how much free space is now left in this block + FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); return (uint8_t*)Addr; } } @@ -83,18 +97,26 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // Save this address as the basis for our next request MemGroup.Near = MB; - MemGroup.PendingMem.push_back(MB); + // Remember that we allocated this memory + MemGroup.AllocatedMem.push_back(MB); Addr = (uintptr_t)MB.base(); uintptr_t EndOfBlock = Addr + MB.size(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + // The allocateMappedMemory may allocate much more memory than we need. In // this case, we store the unused memory as a free memory block. unsigned FreeSize = EndOfBlock-Addr-Size; - if (FreeSize > 16) - MemGroup.FreeMem.push_back(sys::MemoryBlock((void*)(Addr + Size), FreeSize)); + if (FreeSize > 16) { + FreeMemBlock FreeMB; + FreeMB.Free = sys::MemoryBlock((void*)(Addr + Size), FreeSize); + FreeMB.PendingPrefixIndex = (unsigned)-1; + MemGroup.FreeMem.push_back(FreeMB); + } // Return aligned address return (uint8_t*)Addr; @@ -105,9 +127,6 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) // FIXME: Should in-progress permissions be reverted if an error occurs? std::error_code ec; - // Don't allow free memory blocks to be used after setting protection flags. - CodeMem.FreeMem.clear(); - // Make code memory executable. ec = applyMemoryGroupPermissions(CodeMem, sys::Memory::MF_READ | sys::Memory::MF_EXEC); @@ -138,25 +157,52 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) // relocations) will get to the data cache but not to the instruction cache. invalidateInstructionCache(); - // Now, remember that we have successfully applied the permissions to avoid - // having to apply them again. - CodeMem.AllocatedMem.append(CodeMem.PendingMem.begin(),CodeMem.PendingMem.end()); - CodeMem.PendingMem.clear(); + return false; +} - RODataMem.AllocatedMem.append(RODataMem.PendingMem.begin(),RODataMem.PendingMem.end()); - RODataMem.PendingMem.clear(); +static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { + static const size_t PageSize = sys::Process::getPageSize(); - return false; + size_t StartOverlap = + (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; + + size_t TrimmedSize = M.size(); + TrimmedSize -= StartOverlap; + TrimmedSize -= TrimmedSize % PageSize; + + sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), TrimmedSize); + + assert(((uintptr_t)Trimmed.base() % PageSize) == 0); + assert((Trimmed.size() % PageSize) == 0); + assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size()); + + return Trimmed; } + std::error_code SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, unsigned Permissions) { - for (sys::MemoryBlock &MB : MemGroup.PendingMem) if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions)) return EC; + MemGroup.PendingMem.clear(); + + // Now go through free blocks and trim any of them that don't span the entire + // page because one of the pending blocks may have overlapped it. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + FreeMB.Free = trimBlockToPageSize(FreeMB.Free); + // We cleared the PendingMem list, so all these pointers are now invalid + FreeMB.PendingPrefixIndex = (unsigned)-1; + } + + // Remove all blocks which are now empty + MemGroup.FreeMem.erase( + std::remove_if(MemGroup.FreeMem.begin(), MemGroup.FreeMem.end(), + [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }), + MemGroup.FreeMem.end()); + return std::error_code(); } @@ -169,10 +215,7 @@ SectionMemoryManager::~SectionMemoryManager() { for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { for (sys::MemoryBlock &Block : Group->AllocatedMem) sys::Memory::releaseMappedMemory(Block); - for (sys::MemoryBlock &Block : Group->PendingMem) - sys::Memory::releaseMappedMemory(Block); } } } // namespace llvm - diff --git a/llvm/lib/Support/Unix/Memory.inc b/llvm/lib/Support/Unix/Memory.inc index 7bffdf38773..d70319168b8 100644 --- a/llvm/lib/Support/Unix/Memory.inc +++ b/llvm/lib/Support/Unix/Memory.inc @@ -152,6 +152,7 @@ Memory::releaseMappedMemory(MemoryBlock &M) { std::error_code Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { + static const size_t PageSize = Process::getPageSize(); if (M.Address == nullptr || M.Size == 0) return std::error_code(); @@ -160,7 +161,7 @@ Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { int Protect = getPosixProtectionFlags(Flags); - int Result = ::mprotect(M.Address, M.Size, Protect); + int Result = ::mprotect((void*)((uintptr_t)M.Address & ~(PageSize-1)), PageSize*((M.Size+PageSize-1)/PageSize), Protect); if (Result != 0) return std::error_code(errno, std::generic_category()); @@ -180,7 +181,7 @@ Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock, std::string *ErrMsg) { if (NumBytes == 0) return MemoryBlock(); - size_t PageSize = Process::getPageSize(); + static const size_t PageSize = Process::getPageSize(); size_t NumPages = (NumBytes+PageSize-1)/PageSize; int fd = -1; |
