Move objects in the SmallArena and update references.

This will reallocate all objects in the small heap and update
all references that were pointing to this object.

This is not functional yet, there are still references that we are not
tracking at other points in the program, so it's still gated behind the
MOVING_GC flag.

src/gc/collector.cpp

@@ -63,7 +63,7 @@ static std::unordered_set<GCRootHandle*>* getRootHandles() {
     return &root_handles;
 }
 
-static int ncollections = 0;
+int ncollections = 0;
 
 static bool gc_enabled = true;
 static bool should_not_reenter_gc = false;
@@ -388,6 +388,41 @@ void invalidateOrderedFinalizerList() {
     sc_us.log(us);
 }
 
+__attribute__((always_inline)) void visitByGCKind(void* p, GCVisitor& visitor) {
+    assert(((intptr_t)p) % 8 == 0);
+
+    GCAllocation* al = GCAllocation::fromUserData(p);
+    visitor.setSource(al);
+
+    GCKind kind_id = al->kind_id;
+    if (kind_id == GCKind::UNTRACKED) {
+        // Nothing to do here.
+    } else if (kind_id == GCKind::CONSERVATIVE) {
+        uint32_t bytes = al->kind_data;
+        visitor.visitPotentialRange((void**)p, (void**)((char*)p + bytes));
+    } else if (kind_id == GCKind::PRECISE) {
+        uint32_t bytes = al->kind_data;
+        visitor.visitRange((void**)p, (void**)((char*)p + bytes));
+    } else if (kind_id == GCKind::PYTHON) {
+        Box* b = reinterpret_cast<Box*>(p);
+        BoxedClass* cls = b->cls;
+
+        if (cls) {
+            // The cls can be NULL since we use 'new' to construct them.
+            // An arbitrary amount of stuff can happen between the 'new' and
+            // the call to the constructor (ie the args get evaluated), which
+            // can trigger a collection.
+            ASSERT(cls->gc_visit, "%s", getTypeName(b));
+            cls->gc_visit(&visitor, b);
+        }
+    } else if (kind_id == GCKind::RUNTIME) {
+        GCAllocatedRuntime* runtime_obj = reinterpret_cast<GCAllocatedRuntime*>(p);
+        runtime_obj->gc_visit(&visitor);
+    } else {
+        RELEASE_ASSERT(0, "Unhandled kind: %d", (int)kind_id);
+    }
+}
+
 GCRootHandle::GCRootHandle() {
     getRootHandles()->insert(this);
 }
@@ -470,38 +505,9 @@ void GCVisitorPinning::visitPotential(void* p) {
     }
 }
 
-static __attribute__((always_inline)) void visitByGCKind(void* p, GCVisitor& visitor) {
-    assert(((intptr_t)p) % 8 == 0);
-
-    GCAllocation* al = GCAllocation::fromUserData(p);
-    visitor.setSource(al);
-
-    GCKind kind_id = al->kind_id;
-    if (kind_id == GCKind::UNTRACKED) {
-        // Nothing to do here.
-    } else if (kind_id == GCKind::CONSERVATIVE) {
-        uint32_t bytes = al->kind_data;
-        visitor.visitPotentialRange((void**)p, (void**)((char*)p + bytes));
-    } else if (kind_id == GCKind::PRECISE) {
-        uint32_t bytes = al->kind_data;
-        visitor.visitRange((void**)p, (void**)((char*)p + bytes));
-    } else if (kind_id == GCKind::PYTHON) {
-        Box* b = reinterpret_cast<Box*>(p);
-        BoxedClass* cls = b->cls;
-
-        if (cls) {
-            // The cls can be NULL since we use 'new' to construct them.
-            // An arbitrary amount of stuff can happen between the 'new' and
-            // the call to the constructor (ie the args get evaluated), which
-            // can trigger a collection.
-            ASSERT(cls->gc_visit, "%s", getTypeName(b));
-            cls->gc_visit(&visitor, b);
-        }
-    } else if (kind_id == GCKind::RUNTIME) {
-        GCAllocatedRuntime* runtime_obj = reinterpret_cast<GCAllocatedRuntime*>(p);
-        runtime_obj->gc_visit(&visitor);
-    } else {
-        RELEASE_ASSERT(0, "Unhandled kind: %d", (int)kind_id);
+void GCVisitorReplacing::_visit(void** ptr_address) {
+    if (*ptr_address == old_value) {
+        *ptr_address = new_value;
     }
 }
 
@@ -790,14 +796,51 @@ static void mapReferencesPhase(ReferenceMap& refmap) {
     graphTraversalMarking(worklist, visitor);
 }
 
-static void move(ReferenceMap& refmap, GCAllocation* al, size_t size) {
+#define MOVE_LOG 1
+static FILE* move_log;
+
+static void move(ReferenceMap& refmap, GCAllocation* old_al, size_t size) {
+#if MOVE_LOG
+    if (!move_log) {
+        move_log = fopen("movelog.txt", "w");
+    }
+#endif
+
     // Only move objects that are in the reference map (unreachable objects
     // won't be in the reference map).
-    if (refmap.pinned.count(al) == 0 && refmap.references.count(al) > 0) {
-        auto& referencing = refmap.references[al];
+    if (refmap.pinned.count(old_al) == 0 && refmap.references.count(old_al) > 0) {
+        auto& referencing = refmap.references[old_al];
         assert(referencing.size() > 0);
-        // GCAllocation* new_al = realloc(al, size);
-    } else if (refmap.pinned.count(al) == 0) {
+
+        GCAllocation* new_al = global_heap.forceRelocate(old_al);
+        assert(new_al);
+        assert(old_al->user_data != new_al->user_data);
+
+#if MOVE_LOG
+        // Write the moves that have happened to file, for debugging.
+        fprintf(move_log, "%d) %p -> %p\n", ncollections, old_al->user_data, new_al->user_data);
+#endif
+
+        for (GCAllocation* referencer : referencing) {
+            // If the whatever is pointing to the object we just moved has also been moved,
+            // then we need to update the pointer in that moved object.
+            if (refmap.moves.count(referencer) > 0) {
+                referencer = refmap.moves[referencer];
+            }
+
+#if MOVE_LOG
+            fprintf(move_log, "    | referencer %p\n", referencer->user_data);
+#endif
+
+            assert(referencer->kind_id == GCKind::PYTHON || referencer->kind_id == GCKind::PRECISE
+                   || referencer->kind_id == GCKind::RUNTIME);
+            GCVisitorReplacing replacer(old_al->user_data, new_al->user_data);
+            visitByGCKind(referencer->user_data, replacer);
+        }
+
+        assert(refmap.moves.count(old_al) == 0);
+        refmap.moves.emplace(old_al, new_al);
+    } else if (refmap.pinned.count(old_al) == 0) {
         // TODO: This probably should not happen.
     }
 }

src/gc/collector.h

@@ -59,6 +59,8 @@ bool isNonheapRoot(void* p);
 void registerPythonObject(Box* b);
 void invalidateOrderedFinalizerList();
 
+void visitByGCKind(void* p, GCVisitor& visitor);
+
 // Debugging/validation helpers: if a GC should not happen in certain sections (ex during unwinding),
 // use these functions to mark that.  This is different from disableGC/enableGC, since it causes an
 // assert rather than delaying of the next GC.
@@ -103,6 +105,22 @@ public:
     void visitPotential(void* p) MOVING_OVERRIDE;
 };
 
+// Visits the fields and replaces it with new_values if it was equal to old_value.
+class GCVisitorReplacing : public GCVisitor {
+private:
+    void* old_value;
+    void* new_value;
+
+    void _visit(void** p) MOVING_OVERRIDE;
+
+public:
+    GCVisitorReplacing(void* old_value, void* new_value) : old_value(old_value), new_value(new_value) {}
+    virtual ~GCVisitorReplacing() {}
+
+    void visitPotential(void* p) MOVING_OVERRIDE{};
+    void visitPotentialRange(void** start, void** end) MOVING_OVERRIDE{};
+};
+
 class GCAllocation;
 class ReferenceMap {
 public:
@@ -112,6 +130,9 @@ public:
 
     // Map from objects O to all objects that contain a reference to O.
     std::unordered_map<GCAllocation*, std::vector<GCAllocation*>> references;
+
+    // Track movement (reallocation) of objects.
+    std::unordered_map<GCAllocation*, GCAllocation*> moves;
 };
 }
 }

src/gc/heap.cpp

@@ -354,6 +354,26 @@ GCAllocation* SmallArena::realloc(GCAllocation* al, size_t bytes) {
     return rtn;
 }
 
+GCAllocation* SmallArena::forceRelocate(GCAllocation* al) {
+    Block* b = Block::forPointer(al);
+
+    size_t size = b->size;
+
+    // Don't register moves, they don't use more memory and they could trigger another GC.
+    GCAllocation* rtn = alloc(size);
+
+#ifndef NVALGRIND
+    VALGRIND_DISABLE_ERROR_REPORTING;
+    memcpy(rtn, al, size);
+    VALGRIND_ENABLE_ERROR_REPORTING;
+#else
+    memcpy(rtn, al, size);
+#endif
+
+    free(al);
+    return rtn;
+}
+
 void SmallArena::free(GCAllocation* alloc) {
     Block* b = Block::forPointer(alloc);
     size_t size = b->size;
@@ -714,8 +734,6 @@ void SmallArena::_getChainStatistics(HeapStatistics* stats, Block** head) {
 #define LARGE_CHUNK_INDEX(obj, section) (((char*)(obj) - (char*)(section)) >> CHUNK_BITS)
 
 GCAllocation* LargeArena::alloc(size_t size) {
-    registerGCManagedBytes(size);
-
     LOCK_REGION(heap->lock);
 
     // printf ("allocLarge %zu\n", size);
@@ -937,8 +955,6 @@ void LargeArena::_freeLargeObj(LargeObj* obj) {
 
 
 GCAllocation* HugeArena::alloc(size_t size) {
-    registerGCManagedBytes(size);
-
     LOCK_REGION(heap->lock);
 
     size_t total_size = size + sizeof(HugeObj);

src/gc/heap.h

@@ -249,7 +249,6 @@ public:
     }
 
     GCAllocation* alloc(size_t bytes) {
-        registerGCManagedBytes(bytes);
         if (bytes <= 16)
             return _alloc(16, 0);
         else if (bytes <= 32)
@@ -267,6 +266,9 @@ public:
     void forEachReference(std::function<void(GCAllocation*, size_t)>);
 
     GCAllocation* realloc(GCAllocation* alloc, size_t bytes);
+
+    GCAllocation* forceRelocate(GCAllocation* alloc);
+
     void free(GCAllocation* al);
 
     GCAllocation* allocationFrom(void* ptr);
@@ -589,6 +591,7 @@ public:
     }
 
     GCAllocation* alloc(size_t bytes) {
+        registerGCManagedBytes(bytes);
         if (bytes > LargeArena::ALLOC_SIZE_LIMIT)
             return huge_arena.alloc(bytes);
         else if (bytes > sizes[NUM_BUCKETS - 1])
@@ -597,6 +600,24 @@ public:
             return small_arena.alloc(bytes);
     }
 
+    // Slightly different than realloc in that:
+    // 1) The size is the same, so we calls alloc in the SmallArena.
+    // 2) Uses a variant of alloc that doesn't register a change in the number of bytes allocated.
+    // 3) Always reallocates the object to a different address.
+    GCAllocation* forceRelocate(GCAllocation* alloc) {
+        GCAllocation* rtn = NULL;
+        if (large_arena.contains(alloc)) {
+            ASSERT(false, "large arena moves not supported yet");
+        } else if (huge_arena.contains(alloc)) {
+            ASSERT(false, "huge arena moves not supported yet");
+        } else {
+            assert(small_arena.contains(alloc));
+            rtn = small_arena.forceRelocate(alloc);
+        }
+
+        return rtn;
+    }
+
     void destructContents(GCAllocation* alloc);
 
     void free(GCAllocation* alloc) {