Scan pointers used in inline caches and scan relocatable symbols.

src/asm_writing/icinfo.cpp

@@ -17,6 +17,7 @@
 #include <cstring>
 #include <memory>
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/Support/Memory.h"
 
 #include "asm_writing/assembler.h"
@@ -25,6 +26,9 @@
 #include "core/common.h"
 #include "core/options.h"
 #include "core/types.h"
+#include "gc/gc.h"
+#include "gc/heap.h"
+#include "runtime/types.h"
 
 namespace pyston {
 
@@ -82,7 +86,7 @@ uint8_t* ICSlotRewrite::getSlotStart() {
     return (uint8_t*)ic->start_addr + ic_entry->idx * ic->getSlotSize();
 }
 
-void ICSlotRewrite::commit(CommitHook* hook) {
+void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references) {
     bool still_valid = true;
     for (int i = 0; i < dependencies.size(); i++) {
         int orig_version = dependencies[i].second;
@@ -118,6 +122,8 @@ void ICSlotRewrite::commit(CommitHook* hook) {
     // if (VERBOSITY()) printf("Commiting to %p-%p\n", start, start + ic->slot_size);
     memcpy(slot_start, buf, ic->getSlotSize());
 
+    ic_entry->gc_references = std::move(gc_references);
+
     ic->times_rewritten++;
 
     if (ic->times_rewritten == IC_MEGAMORPHIC_THRESHOLD) {
@@ -187,6 +193,25 @@ ICSlotInfo* ICInfo::pickEntryForRewrite(const char* debug_name) {
     return NULL;
 }
 
+// Keep track of all ICInfo(s) that we create because they contain pointers to Pyston heap objects
+// that we have written into the generated code and we may need to scan those.
+static llvm::DenseSet<ICInfo*> ics_list;
+static llvm::DenseMap<void*, ICInfo*> ics_by_return_addr;
+
+void registerGCTrackedICInfo(ICInfo* ic) {
+#if MOVING_GC
+    assert(ics_list.count(ic) == 0);
+    ics_list.insert(ic);
+#endif
+}
+
+void deregisterGCTrackedICInfo(ICInfo* ic) {
+#if MOVING_GC
+    assert(ics_list.count(ic) == 1);
+    ics_list.erase(ic);
+#endif
+}
+
 ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
                int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet _live_outs,
                assembler::GenericRegister return_register, TypeRecorder* type_recorder)
@@ -207,9 +232,18 @@ ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, S
     for (int i = 0; i < num_slots; i++) {
         slots.emplace_back(this, i);
     }
+
+#if MOVING_GC
+    assert(ics_list.count(this) == 0);
+#endif
+}
+
+ICInfo::~ICInfo() {
+#if MOVING_GC
+    assert(ics_list.count(this) == 0);
+#endif
 }
 
-static llvm::DenseMap<void*, ICInfo*> ics_by_return_addr;
 std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr,
                                                    uint8_t* continue_addr, uint8_t* slowpath_rtn_addr,
                                                    const ICSetupInfo* ic, StackInfo stack_info, LiveOutSet live_outs) {
@@ -251,14 +285,19 @@ std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t*
     ICInfo* icinfo = new ICInfo(start_addr, slowpath_rtn_addr, continue_addr, stack_info, ic->num_slots, ic->slot_size,
                                 ic->getCallingConvention(), std::move(live_outs), return_register, ic->type_recorder);
 
+    assert(!ics_by_return_addr.count(slowpath_rtn_addr));
     ics_by_return_addr[slowpath_rtn_addr] = icinfo;
 
+    registerGCTrackedICInfo(icinfo);
+
     return std::unique_ptr<ICInfo>(icinfo);
 }
 
 void deregisterCompiledPatchpoint(ICInfo* ic) {
     assert(ics_by_return_addr.count(ic->slowpath_rtn_addr));
     ics_by_return_addr.erase(ic->slowpath_rtn_addr);
+
+    deregisterGCTrackedICInfo(ic);
 }
 
 ICInfo* getICInfo(void* rtn_addr) {
@@ -305,4 +344,12 @@ bool ICInfo::shouldAttempt() {
 bool ICInfo::isMegamorphic() {
     return times_rewritten >= IC_MEGAMORPHIC_THRESHOLD;
 }
+
+void ICInfo::visitGCReferences(gc::GCVisitor* v) {
+    for (const auto& p : ics_list) {
+        for (auto& slot : p->slots) {
+            v->visitNonRelocatableRange(&slot.gc_references[0], &slot.gc_references[slot.gc_references.size()]);
+        }
+    }
+}
 }

src/asm_writing/icinfo.h

@@ -27,6 +27,10 @@
 
 namespace pyston {
 
+namespace gc {
+class GCVisitor;
+}
+
 class TypeRecorder;
 
 class ICInfo;
@@ -43,6 +47,8 @@ public:
     int idx;        // the index inside the ic
     int num_inside; // the number of stack frames that are currently inside this slot
 
+    std::vector<void*> gc_references;
+
     void clear();
 };
 
@@ -85,7 +91,7 @@ public:
     void gc_visit(gc::GCVisitor* visitor);
 
     void addDependenceOn(ICInvalidator&);
-    void commit(CommitHook* hook);
+    void commit(CommitHook* hook, std::vector<void*> gc_references);
     void abort();
 
     const ICInfo* getICInfo() { return ic; }
@@ -124,6 +130,7 @@ public:
     ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
            int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet live_outs,
            assembler::GenericRegister return_register, TypeRecorder* type_recorder);
+    ~ICInfo();
     void* const start_addr, *const slowpath_rtn_addr, *const continue_addr;
 
     int getSlotSize() { return slot_size; }
@@ -142,8 +149,12 @@ public:
     int percentBackedoff() const { return retry_backoff; }
 
     friend class ICSlotRewrite;
+    static void visitGCReferences(gc::GCVisitor* visitor);
 };
 
+void registerGCTrackedICInfo(ICInfo* ic);
+void deregisterGCTrackedICInfo(ICInfo* ic);
+
 class ICSetupInfo;
 struct CompiledFunction;
 std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr,

src/asm_writing/rewriter.cpp

@@ -19,6 +19,7 @@
 #include "codegen/unwinding.h"
 #include "core/common.h"
 #include "core/stats.h"
+#include "gc/heap.h"
 #include "runtime/types.h"
 
 namespace pyston {
@@ -350,6 +351,7 @@ void RewriterVar::addAttrGuard(int offset, uint64_t val, bool negate) {
 
     if (!attr_guards.insert(std::make_tuple(offset, val, negate)).second)
         return; // duplicate guard detected
+
     RewriterVar* val_var = rewriter->loadConst(val);
     rewriter->addAction([=]() { rewriter->_addAttrGuard(this, offset, val_var, negate); }, { this, val_var },
                         ActionType::GUARD);
@@ -717,6 +719,15 @@ void Rewriter::_trap() {
 RewriterVar* Rewriter::loadConst(int64_t val, Location dest) {
     STAT_TIMER(t0, "us_timer_rewriter", 10);
 
+#if MOVING_GC
+    // Moving GCs need to know all existing references in the program, including those that end
+    // up in compiled code.
+    auto al = gc::global_heap.getAllocationFromInteriorPointer((void*)val);
+    if (al) {
+        gc_references.push_back(al->user_data);
+    }
+#endif
+
     for (auto& p : const_loader.consts) {
         if (p.first != val)
             continue;
@@ -1328,7 +1339,7 @@ void Rewriter::commit() {
     }
 #endif
 
-    rewrite->commit(this);
+    rewrite->commit(this, std::move(gc_references));
 
     if (assembler->hasFailed()) {
         on_assemblyfail();

src/asm_writing/rewriter.h

@@ -442,6 +442,7 @@ protected:
     }
     bool added_changing_action;
     bool marked_inside_ic;
+    std::vector<void*> gc_references;
 
     bool done_guarding;
     bool isDoneGuarding() {

src/codegen/baseline_jit.cpp

@@ -624,6 +624,15 @@ int JitFragmentWriter::finishCompilation() {
 
     void* next_fragment_start = (uint8_t*)block->code + assembler->bytesWritten();
     code_block.fragmentFinished(assembler->bytesWritten(), num_bytes_overlapping, next_fragment_start);
+
+#if MOVING_GC
+    // If JitFragmentWriter is destroyed, we don't necessarily want the ICInfo to be destroyed also,
+    // because it may contain a list of references to pointers in generated code that still exists
+    // and we need to keep those around.
+    // TODO: When should these ICInfo be freed?
+    registerGCTrackedICInfo(ic_info.release());
+#endif
+
     return num_bytes_exit;
 }
 

src/codegen/irgen/util.cpp

@@ -27,6 +27,7 @@
 #include "codegen/codegen.h"
 #include "codegen/patchpoints.h"
 #include "core/common.h"
+#include "gc/gc.h"
 #include "runtime/types.h"
 
 namespace pyston {
@@ -97,7 +98,7 @@ llvm::Constant* getStringConstantPtr(llvm::StringRef str) {
 // It's slightly easier to emit them as integers (there are primitive integer constants but not pointer constants),
 // but doing it this way makes it clearer what's going on.
 
-static llvm::StringMap<const void*> relocatable_syms;
+llvm::StringMap<const void*> relocatable_syms;
 
 void clearRelocatableSymsMap() {
     relocatable_syms.clear();

src/codegen/irgen/util.h

@@ -17,6 +17,7 @@
 
 #include <string>
 
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace llvm {
@@ -25,6 +26,10 @@ class Function;
 class Type;
 }
 
+namespace gc {
+class GCVisitor;
+}
+
 namespace pyston {
 
 llvm::Constant* embedRelocatablePtr(const void* addr, llvm::Type*, llvm::StringRef shared_name = llvm::StringRef());
@@ -37,6 +42,8 @@ llvm::Constant* getNullPtr(llvm::Type* t);
 void clearRelocatableSymsMap();
 const void* getValueOfRelocatableSym(const std::string& str);
 
+void visitRelocatableSymsMap(gc::GCVisitor* visitor);
+
 void dumpPrettyIR(llvm::Function* f);
 }
 

src/gc/collector.cpp

@@ -18,8 +18,10 @@
 #include <cstdio>
 #include <cstdlib>
 
+#include "asm_writing/icinfo.h"
 #include "codegen/ast_interpreter.h"
 #include "codegen/codegen.h"
+#include "codegen/irgen/util.h"
 #include "core/common.h"
 #include "core/threading.h"
 #include "core/types.h"
@@ -429,6 +431,11 @@ static void markRoots(GCVisitor& visitor) {
     for (auto weakref : weakrefs_needing_callback_list) {
         visitor.visit(weakref);
     }
+
+    GC_TRACE_LOG("Looking at generated code pointers\n");
+#if MOVING_GC
+    ICInfo::visitGCReferences(&visitor);
+#endif
 }
 
 static void finalizationOrderingFindReachable(Box* obj) {

src/gc/gc.h

@@ -86,6 +86,14 @@ public:
     virtual void visitRedundantRange(void** start, void** end) {}
     virtual void visitPotentialRedundant(void* p) {}
     virtual void visitPotentialRangeRedundant(void* const* start, void* const* end) {}
+
+    // Visit pointers to objects that we know cannot be moved.
+    // This is often used to scan a pointer that's a copy of a pointer stored in a place that
+    // we cannot easily scanned (like generated code).
+    // This default to visitPotential for now (which also cannot be moved) but we may want to
+    // change that later for performance.
+    void visitNonRelocatable(void* p) { visitPotential(p); }
+    void visitNonRelocatableRange(void** start, void** end) { visitPotentialRange(start, end); }
 };
 
 enum class GCKind : uint8_t {