Merge pull request #1365 from corona10/section

Change section memory manager to fix performance problem in long-runn…

llvm_patches/0014-PATCH-SectionMemoryManger-Make-better-use-of-virtual.patch

+From b7035692e3abd5d5b6781d59d17f8a5cf98297f9 Mon Sep 17 00:00:00 2001
+From: Dong-hee Na <corona10@gmail.com>
+Date: Tue, 20 Sep 2016 23:11:35 +0900
+Subject: [PATCH] [PATCH] SectionMemoryManger: Make better use of virtual
+ memory
+
+---
+ lib/Support/Unix/Memory.inc | 19 ++++++++-----------
+ 1 file changed, 8 insertions(+), 11 deletions(-)
+
+diff --git a/lib/Support/Unix/Memory.inc b/lib/Support/Unix/Memory.inc
+index c421ee8..f3463e5 100644
+--- a/lib/Support/Unix/Memory.inc
++++ b/lib/Support/Unix/Memory.inc
+@@ -50,9 +50,8 @@ int getPosixProtectionFlags(unsigned Flags) {
+     return PROT_READ | PROT_WRITE;
+   case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_EXEC:
+     return PROT_READ | PROT_EXEC;
+-  case llvm::sys::Memory::MF_READ |
+-	 llvm::sys::Memory::MF_WRITE |
+-	 llvm::sys::Memory::MF_EXEC:
++  case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE |
++      llvm::sys::Memory::MF_EXEC:
+     return PROT_READ | PROT_WRITE | PROT_EXEC;
+   case llvm::sys::Memory::MF_EXEC:
+ #if defined(__FreeBSD__)
+@@ -74,7 +73,7 @@ int getPosixProtectionFlags(unsigned Flags) {
+   return PROT_NONE;
+ }
+ 
+-} // namespace
++} // anonymous namespace
+ 
+ namespace llvm {
+ namespace sys {
+@@ -153,6 +152,7 @@ Memory::releaseMappedMemory(MemoryBlock &M) {
+ 
+ std::error_code
+ Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) {
++  static const size_t PageSize = Process::getPageSize();
+   if (M.Address == nullptr || M.Size == 0)
+     return std::error_code();
+ 
+@@ -161,7 +161,7 @@ Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) {
+ 
+   int Protect = getPosixProtectionFlags(Flags);
+ 
+-  int Result = ::mprotect(M.Address, M.Size, Protect);
++  int Result = ::mprotect((void*)((uintptr_t)M.Address & ~(PageSize-1)), PageSize*((M.Size+PageSize-1)/PageSize), Protect);
+   if (Result != 0)
+     return std::error_code(errno, std::generic_category());
+ 
+@@ -181,7 +181,7 @@ Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock,
+                     std::string *ErrMsg) {
+   if (NumBytes == 0) return MemoryBlock();
+ 
+-  size_t PageSize = Process::getPageSize();
++  static const size_t PageSize = Process::getPageSize();
+   size_t NumPages = (NumBytes+PageSize-1)/PageSize;
+ 
+   int fd = -1;
+@@ -265,15 +265,12 @@ bool Memory::setWritable (MemoryBlock &M, std::string *ErrMsg) {
+ }
+ 
+ bool Memory::setExecutable (MemoryBlock &M, std::string *ErrMsg) {
+-#if defined(__APPLE__) && (defined(__arm__) || defined(__arm64__))
+-  if (M.Address == 0 || M.Size == 0) return false;
++  if (M.Address == nullptr || M.Size == 0) return false;
+   Memory::InvalidateInstructionCache(M.Address, M.Size);
++#if defined(__APPLE__) && (defined(__arm__) || defined(__arm64__))
+   kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)M.Address,
+     (vm_size_t)M.Size, 0, VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY);
+   return KERN_SUCCESS == kr;
+-#elif defined(__arm__) || defined(__aarch64__)
+-  Memory::InvalidateInstructionCache(M.Address, M.Size);
+-  return true;
+ #else
+   return true;
+ #endif
+-- 
+2.7.4
+

src/codegen/memmgr.cpp

@@ -19,6 +19,7 @@
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Memory.h"
+#include "llvm/Support/Process.h"
 
 #include "codegen/irgen/util.h"
 #include "codegen/unwinding.h"
@@ -49,9 +50,14 @@ public:
 private:
     void invalidateInstructionCache();
 
+    struct FreeMemBlock {
+        sys::MemoryBlock Free;
+        unsigned PendingPrefixIndex;
+    };
     struct MemoryGroup {
+        SmallVector<sys::MemoryBlock, 16> PendingMem;
         SmallVector<sys::MemoryBlock, 16> AllocatedMem;
-        SmallVector<sys::MemoryBlock, 16> FreeMem;
+        SmallVector<FreeMemBlock, 16> FreeMem;
         sys::MemoryBlock Near;
     };
 
@@ -93,19 +99,30 @@ uint8_t* PystonMemoryManager::allocateSection(MemoryGroup& MemGroup, uintptr_t S
 
     // Look in the list of free memory regions and use a block there if one
     // is available.
-    for (int i = 0, e = MemGroup.FreeMem.size(); i != e; ++i) {
-        sys::MemoryBlock& MB = MemGroup.FreeMem[i];
-        if (MB.size() >= RequiredSize) {
-            Addr = (uintptr_t)MB.base();
-            uintptr_t EndOfBlock = Addr + MB.size();
+    for (FreeMemBlock& FreeMB : MemGroup.FreeMem) {
+        if (FreeMB.Free.size() >= RequiredSize) {
+            Addr = (uintptr_t)FreeMB.Free.base();
+            uintptr_t EndOfBlock = Addr + FreeMB.Free.size();
             // Align the address.
             Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
-            // Store cutted free memory block.
-            MemGroup.FreeMem[i] = sys::MemoryBlock((void*)(Addr + Size), EndOfBlock - Addr - Size);
+
+            if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
+                // The part of the block we're giving out to the user is now pending
+                MemGroup.PendingMem.push_back(sys::MemoryBlock((void*)Addr, Size));
+
+                // Remember this pending block, such that future allocations can just
+                // modify it rather than creating a new one
+                FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
+            } else {
+                sys::MemoryBlock& PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
+                PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base());
+            }
+
+            // Remember how much free space is now left in this block
+            FreeMB.Free = sys::MemoryBlock((void*)(Addr + Size), EndOfBlock - Addr - Size);
             return (uint8_t*)Addr;
         }
     }
-
     // No pre-allocated free block was large enough. Allocate a new memory region.
     // Note that all sections get allocated as read-write.  The permissions will
     // be updated later based on memory group.
@@ -120,7 +137,7 @@ uint8_t* PystonMemoryManager::allocateSection(MemoryGroup& MemGroup, uintptr_t S
                                                             sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec);
     if (ec) {
         // FIXME: Add error propogation to the interface.
-        return NULL;
+        return nullptr;
     }
 
     std::string stat_name = "mem_section_" + std::string(SectionName);
@@ -129,6 +146,7 @@ uint8_t* PystonMemoryManager::allocateSection(MemoryGroup& MemGroup, uintptr_t S
     // Save this address as the basis for our next request
     MemGroup.Near = MB;
 
+    // Remember that we allocated this memory
     MemGroup.AllocatedMem.push_back(MB);
     Addr = (uintptr_t)MB.base();
     uintptr_t EndOfBlock = Addr + MB.size();
@@ -136,11 +154,18 @@ uint8_t* PystonMemoryManager::allocateSection(MemoryGroup& MemGroup, uintptr_t S
     // Align the address.
     Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
 
+    // The part of the block we're giving out to the user is now pending
+    MemGroup.PendingMem.push_back(sys::MemoryBlock((void*)Addr, Size));
+
     // The allocateMappedMemory may allocate much more memory than we need. In
     // this case, we store the unused memory as a free memory block.
     unsigned FreeSize = EndOfBlock - Addr - Size;
-    if (FreeSize > 16)
-        MemGroup.FreeMem.push_back(sys::MemoryBlock((void*)(Addr + Size), FreeSize));
+    if (FreeSize > 16) {
+        FreeMemBlock FreeMB;
+        FreeMB.Free = sys::MemoryBlock((void*)(Addr + Size), FreeSize);
+        FreeMB.PendingPrefixIndex = (unsigned)-1;
+        MemGroup.FreeMem.push_back(FreeMB);
+    }
 
     // Return aligned address
     return (uint8_t*)Addr;
@@ -150,9 +175,6 @@ bool PystonMemoryManager::finalizeMemory(std::string* ErrMsg) {
     // FIXME: Should in-progress permissions be reverted if an error occurs?
     llvm_error_code ec;
 
-    // Don't allow free memory blocks to be used after setting protection flags.
-    CodeMem.FreeMem.clear();
-
     // Make code memory executable.
     // pyston: also make it writeable so we can patch it later
     ec = applyMemoryGroupPermissions(CodeMem, sys::Memory::MF_READ | sys::Memory::MF_EXEC | sys::Memory::MF_WRITE);
@@ -160,19 +182,16 @@ bool PystonMemoryManager::finalizeMemory(std::string* ErrMsg) {
         if (ErrMsg) {
             *ErrMsg = ec.message();
         }
-        return true;
+        RELEASE_ASSERT(0, "finalizeMemory failed");
     }
 
-    // Don't allow free memory blocks to be used after setting protection flags.
-    RODataMem.FreeMem.clear();
-
     // Make read-only data memory read-only.
     ec = applyMemoryGroupPermissions(RODataMem, sys::Memory::MF_READ | sys::Memory::MF_EXEC);
     if (ec) {
         if (ErrMsg) {
             *ErrMsg = ec.message();
         }
-        return true;
+        RELEASE_ASSERT(0, "finalizeMemory failed");
     }
 
     // Read-write data memory already has the correct permissions
@@ -185,15 +204,46 @@ bool PystonMemoryManager::finalizeMemory(std::string* ErrMsg) {
     return false;
 }
 
+static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
+    static const size_t PageSize = sys::Process::getPageSize();
+
+    size_t StartOverlap = (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
+
+    size_t TrimmedSize = M.size();
+    TrimmedSize -= StartOverlap;
+    TrimmedSize -= TrimmedSize % PageSize;
+
+    sys::MemoryBlock Trimmed((void*)((uintptr_t)M.base() + StartOverlap), TrimmedSize);
+
+    assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
+    assert((Trimmed.size() % PageSize) == 0);
+    assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size());
+
+    return Trimmed;
+}
+
 llvm_error_code PystonMemoryManager::applyMemoryGroupPermissions(MemoryGroup& MemGroup, unsigned Permissions) {
 
-    for (int i = 0, e = MemGroup.AllocatedMem.size(); i != e; ++i) {
+    for (sys::MemoryBlock& MB : MemGroup.PendingMem) {
         llvm_error_code ec;
-        ec = sys::Memory::protectMappedMemory(MemGroup.AllocatedMem[i], Permissions);
+        ec = sys::Memory::protectMappedMemory(MB, Permissions);
         if (ec) {
             return ec;
         }
     }
+    MemGroup.PendingMem.clear();
+
+    // Now go through free blocks and trim any of them that don't span the entire
+    // page because one of the pending blocks may have overlapped it.
+    for (FreeMemBlock& FreeMB : MemGroup.FreeMem) {
+        FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
+        // We cleared the PendingMem list, so all these pointers are now invalid
+        FreeMB.PendingPrefixIndex = (unsigned)-1;
+    }
+
+    // Remove all blocks which are now empty
+    MemGroup.FreeMem.erase(remove_if(MemGroup.FreeMem, [](FreeMemBlock& FreeMB) { return FreeMB.Free.size() == 0; }),
+                           MemGroup.FreeMem.end());
 
 #if LLVMREV < 209952
     return llvm_error_code::success();
@@ -203,8 +253,8 @@ llvm_error_code PystonMemoryManager::applyMemoryGroupPermissions(MemoryGroup& Me
 }
 
 void PystonMemoryManager::invalidateInstructionCache() {
-    for (int i = 0, e = CodeMem.AllocatedMem.size(); i != e; ++i)
-        sys::Memory::InvalidateInstructionCache(CodeMem.AllocatedMem[i].base(), CodeMem.AllocatedMem[i].size());
+    for (sys::MemoryBlock& Block : CodeMem.PendingMem)
+        sys::Memory::InvalidateInstructionCache(Block.base(), Block.size());
 }
 
 uint64_t PystonMemoryManager::getSymbolAddress(const std::string& name) {
@@ -230,12 +280,10 @@ uint64_t PystonMemoryManager::getSymbolAddress(const std::string& name) {
 }
 
 PystonMemoryManager::~PystonMemoryManager() {
-    for (unsigned i = 0, e = CodeMem.AllocatedMem.size(); i != e; ++i)
-        sys::Memory::releaseMappedMemory(CodeMem.AllocatedMem[i]);
-    for (unsigned i = 0, e = RWDataMem.AllocatedMem.size(); i != e; ++i)
-        sys::Memory::releaseMappedMemory(RWDataMem.AllocatedMem[i]);
-    for (unsigned i = 0, e = RODataMem.AllocatedMem.size(); i != e; ++i)
-        sys::Memory::releaseMappedMemory(RODataMem.AllocatedMem[i]);
+    for (MemoryGroup* Group : { &CodeMem, &RWDataMem, &RODataMem }) {
+        for (sys::MemoryBlock& Block : Group->AllocatedMem)
+            sys::Memory::releaseMappedMemory(Block);
+    }
 }
 
 std::unique_ptr<llvm::RTDyldMemoryManager> createMemoryManager() {