Merge PR #661 from toshok:binary-search-large-and-huge-arenas

Conflicts:
	src/core/types.h

Makefile

@@ -530,7 +530,8 @@ check:
 # Travis-CI do the full test.
 .PHONY: quick_check
 quick_check:
-	$(MAKE) pyston_dbg check-deps
+	$(MAKE) pyston_dbg
+	$(MAKE) check-deps
 	( cd $(CMAKE_DIR_DBG) && ctest -V -R 'check-format|unittests|pyston_defaults_tests|pyston_defaults_cpython' )
 
 

src/codegen/unwinding.cpp

@@ -118,35 +118,22 @@ void registerDynamicEhFrame(uint64_t code_addr, size_t code_size, uint64_t eh_fr
     // dyn_info that contains a binary search table.
 }
 
+struct compare_cf {
+    int operator()(const uint64_t& key, const CompiledFunction* item) const {
+        // key is the return address of the callsite, so we will check it against
+        // the region (start, end] (opposite-endedness of normal half-open regions)
+        if (key <= item->code_start)
+            return -1;
+        else if (key > item->code_start + item->code_size)
+            return 1;
+        return 0;
+    }
+};
+
 class CFRegistry {
 private:
     std::vector<CompiledFunction*> cfs;
 
-    // similar to Java's Array.binarySearch:
-    // return values are either:
-    //   >= 0 : the index where a given item was found
-    //   < 0  : a negative number that can be transformed (using "-num-1") into the insertion point
-    //
-    // addr is the return address of the callsite, so we will check it against
-    // the region (start, end] (opposite-endedness of normal half-open regions)
-    //
-    int find_cf(uint64_t addr) {
-        int l = 0;
-        int r = cfs.size() - 1;
-        while (l <= r) {
-            int mid = l + (r - l) / 2;
-            auto mid_cf = cfs[mid];
-            if (addr <= mid_cf->code_start) {
-                r = mid - 1;
-            } else if (addr > mid_cf->code_start + mid_cf->code_size) {
-                l = mid + 1;
-            } else {
-                return mid;
-            }
-        }
-        return -(l + 1);
-    }
-
 public:
     void registerCF(CompiledFunction* cf) {
         if (cfs.empty()) {
@@ -154,7 +141,7 @@ public:
             return;
         }
 
-        int idx = find_cf((uint64_t)cf->code_start);
+        int idx = binarySearch((uint64_t)cf->code_start, cfs.begin(), cfs.end(), compare_cf());
         if (idx >= 0)
             RELEASE_ASSERT(0, "CompiledFunction registered twice?");
 
@@ -165,7 +152,7 @@ public:
         if (cfs.empty())
             return NULL;
 
-        int idx = find_cf(addr);
+        int idx = binarySearch(addr, cfs.begin(), cfs.end(), compare_cf());
         if (idx >= 0)
             return cfs[idx];
 

src/core/types.h

@@ -749,6 +749,29 @@ struct CallattrFlags {
     uint64_t asInt() { return (uint64_t(argspec.asInt()) << 32) | (cls_only << 0) | (null_on_nonexistent << 1); }
 };
 static_assert(sizeof(CallattrFlags) == sizeof(uint64_t), "");
+
+// similar to Java's Array.binarySearch:
+// return values are either:
+//   >= 0 : the index where a given item was found
+//   < 0  : a negative number that can be transformed (using "-num-1") into the insertion point
+//
+template <typename T, typename RandomAccessIterator, typename Cmp>
+int binarySearch(T needle, RandomAccessIterator start, RandomAccessIterator end, Cmp cmp) {
+    int l = 0;
+    int r = end - start - 1;
+    while (l <= r) {
+        int mid = l + (r - l) / 2;
+        auto mid_item = *(start + mid);
+        int c = cmp(needle, mid_item);
+        if (c < 0)
+            r = mid - 1;
+        else if (c > 0)
+            l = mid + 1;
+        else
+            return mid;
+    }
+    return -(l + 1);
+}
 }
 
 namespace std {

src/gc/collector.cpp

@@ -438,6 +438,8 @@ void runCollection() {
 
     Timer _t("collecting", /*min_usec=*/10000);
 
+    global_heap.prepareForCollection();
+
     markPhase();
 
     // The sweep phase will not free weakly-referenced objects, so that we can inspect their
@@ -500,6 +502,8 @@ void runCollection() {
         }
     }
 
+    global_heap.cleanupAfterCollection();
+
     if (VERBOSITY("gc") >= 2)
         printf("Collection #%d done\n\n", ncollections);
 

src/gc/heap.cpp

@@ -661,17 +661,46 @@ void LargeArena::free(GCAllocation* al) {
     _freeLargeObj(LargeObj::fromAllocation(al));
 }
 
+struct CompareObjLookupCache {
+    int operator()(const void* p, const ObjLookupCache& obj) {
+        if (p < (char*)obj.data)
+            return -1;
+        if (p >= (char*)obj.data + obj.size)
+            return 1;
+        return 0;
+    }
+};
+
 GCAllocation* LargeArena::allocationFrom(void* ptr) {
-    LargeObj* obj = NULL;
+    if (lookup.size()) {
+        int idx = binarySearch(ptr, lookup.begin(), lookup.end(), CompareObjLookupCache());
+        if (idx < 0)
+            return NULL;
+        return (GCAllocation*)lookup[idx].data;
+    } else {
+        LargeObj* obj = NULL;
 
-    for (obj = head; obj; obj = obj->next) {
-        char* end = (char*)&obj->data + obj->size;
+        for (obj = head; obj; obj = obj->next) {
+            char* end = (char*)&obj->data + obj->size;
 
-        if (ptr >= obj->data && ptr < end) {
-            return &obj->data[0];
+            if (ptr >= obj->data && ptr < end) {
+                return &obj->data[0];
+            }
         }
+        return NULL;
     }
-    return NULL;
+}
+
+void LargeArena::prepareForCollection() {
+    for (LargeObj* lo = head; lo; lo = lo->next) {
+        lookup.push_back(ObjLookupCache(lo, &lo->data[0], lo->size));
+    }
+    std::sort(lookup.begin(), lookup.end(),
+              [](const ObjLookupCache& lo1, const ObjLookupCache& lo2) { return lo1.data < lo2.data; });
+}
+
+void LargeArena::cleanupAfterCollection() {
+    lookup.clear();
 }
 
 void LargeArena::freeUnmarked(std::vector<Box*>& weakly_referenced, std::vector<BoxedClass*>& classes_to_free) {
@@ -682,7 +711,6 @@ void LargeArena::getStatistics(HeapStatistics* stats) {
     forEach(head, [stats](LargeObj* obj) { addStatistic(stats, obj->data, obj->size); });
 }
 
-
 void LargeArena::add_free_chunk(LargeFreeChunk* free_chunks, size_t size) {
     size_t num_chunks = size >> CHUNK_BITS;
 
@@ -831,7 +859,7 @@ GCAllocation* HugeArena::alloc(size_t size) {
     total_size = (total_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
     extendMapping(total_size);
     HugeObj* rtn = (HugeObj*)allocFromArena(total_size);
-    rtn->obj_size = size;
+    rtn->size = size;
 
     nullNextPrev(rtn);
     insertIntoLL(&head, rtn);
@@ -847,7 +875,7 @@ GCAllocation* HugeArena::realloc(GCAllocation* al, size_t bytes) {
         return al;
 
     GCAllocation* rtn = heap->alloc(bytes);
-    memcpy(rtn, al, std::min(bytes, obj->obj_size));
+    memcpy(rtn, al, std::min(bytes, obj->size));
 
     _freeHugeObj(obj);
     return rtn;
@@ -858,13 +886,32 @@ void HugeArena::free(GCAllocation* al) {
 }
 
 GCAllocation* HugeArena::allocationFrom(void* ptr) {
-    HugeObj* cur = head;
-    while (cur) {
-        if (ptr >= cur && ptr < &cur->data[cur->obj_size])
-            return &cur->data[0];
-        cur = cur->next;
+    if (lookup.size()) {
+        int idx = binarySearch(ptr, lookup.begin(), lookup.end(), CompareObjLookupCache());
+        if (idx < 0)
+            return NULL;
+        return (GCAllocation*)lookup[idx].data;
+    } else {
+        HugeObj* cur = head;
+        while (cur) {
+            if (ptr >= cur && ptr < &cur->data[cur->size])
+                return &cur->data[0];
+            cur = cur->next;
+        }
+        return NULL;
+    }
+}
+
+void HugeArena::prepareForCollection() {
+    for (HugeObj* lo = head; lo; lo = lo->next) {
+        lookup.push_back(ObjLookupCache(lo, &lo->data[0], lo->size));
     }
-    return NULL;
+    std::sort(lookup.begin(), lookup.end(),
+              [](const ObjLookupCache& lo1, const ObjLookupCache& lo2) { return lo1.data < lo2.data; });
+}
+
+void HugeArena::cleanupAfterCollection() {
+    lookup.clear();
 }
 
 void HugeArena::freeUnmarked(std::vector<Box*>& weakly_referenced, std::vector<BoxedClass*>& classes_to_free) {

src/gc/heap.h

@@ -227,6 +227,9 @@ public:
 
     void getStatistics(HeapStatistics* stats);
 
+    void prepareForCollection() {}
+    void cleanupAfterCollection() {}
+
 private:
     template <int N> class Bitmap {
         static_assert(N % 64 == 0, "");
@@ -361,6 +364,15 @@ private:
     GCAllocation* __attribute__((__malloc__)) _alloc(size_t bytes, int bucket_idx);
 };
 
+struct ObjLookupCache {
+    void* objptr;
+    void* data;
+    size_t size;
+
+    ObjLookupCache(void* objptr, void* data, size_t size) : objptr(objptr), data(data), size(size) {}
+};
+
+
 //
 // The LargeArena allocates objects where 3584 < size <1024*1024-CHUNK_SIZE-sizeof(LargeObject) bytes.
 //
@@ -407,6 +419,7 @@ private:
 
     static constexpr int NUM_FREE_LISTS = 32;
 
+    std::vector<ObjLookupCache> lookup; // used during gc's to speed up finding large object GCAllocations
     Heap* heap;
     LargeObj* head;
     LargeBlock* blocks;
@@ -431,6 +444,9 @@ public:
     void freeUnmarked(std::vector<Box*>& weakly_referenced, std::vector<BoxedClass*>& classes_to_free);
 
     void getStatistics(HeapStatistics* stats);
+
+    void prepareForCollection();
+    void cleanupAfterCollection();
 };
 
 // The HugeArena allocates objects where size > 1024*1024 bytes.
@@ -450,14 +466,17 @@ public:
 
     void getStatistics(HeapStatistics* stats);
 
+    void prepareForCollection();
+    void cleanupAfterCollection();
+
 private:
     struct HugeObj {
         HugeObj* next, **prev;
-        size_t obj_size;
+        size_t size;
         GCAllocation data[0];
 
         int mmap_size() {
-            size_t total_size = obj_size + sizeof(HugeObj);
+            size_t total_size = size + sizeof(HugeObj);
             total_size = (total_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
             return total_size;
         }
@@ -474,6 +493,7 @@ private:
     void _freeHugeObj(HugeObj* lobj);
 
     HugeObj* head;
+    std::vector<ObjLookupCache> lookup; // used during gc's to speed up finding large object GCAllocations
 
     Heap* heap;
 };
@@ -552,6 +572,18 @@ public:
         huge_arena.freeUnmarked(weakly_referenced, classes_to_free);
     }
 
+    void prepareForCollection() {
+        small_arena.prepareForCollection();
+        large_arena.prepareForCollection();
+        huge_arena.prepareForCollection();
+    }
+
+    void cleanupAfterCollection() {
+        small_arena.cleanupAfterCollection();
+        large_arena.cleanupAfterCollection();
+        huge_arena.cleanupAfterCollection();
+    }
+
     void dumpHeapStatistics(int level);
 
 private: