Merge pull request #885 from rudi-c/compileconstants

Scan constants written into compiled code.

src/asm_writing/icinfo.cpp

@@ -17,6 +17,7 @@
 #include <cstring>
 #include <memory>
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/Support/Memory.h"
 
 #include "asm_writing/assembler.h"
@@ -25,6 +26,9 @@
 #include "core/common.h"
 #include "core/options.h"
 #include "core/types.h"
+#include "gc/gc.h"
+#include "gc/heap.h"
+#include "runtime/types.h"
 
 namespace pyston {
 
@@ -49,8 +53,6 @@ void ICInvalidator::invalidateAll() {
     dependents.clear();
 }
 
-
-
 void ICSlotInfo::clear() {
     ic->clear(this);
 }
@@ -82,7 +84,7 @@ uint8_t* ICSlotRewrite::getSlotStart() {
     return (uint8_t*)ic->start_addr + ic_entry->idx * ic->getSlotSize();
 }
 
-void ICSlotRewrite::commit(CommitHook* hook) {
+void ICSlotRewrite::commit(CommitHook* hook, std::vector<void*> gc_references) {
     bool still_valid = true;
     for (int i = 0; i < dependencies.size(); i++) {
         int orig_version = dependencies[i].second;
@@ -118,6 +120,8 @@ void ICSlotRewrite::commit(CommitHook* hook) {
     // if (VERBOSITY()) printf("Commiting to %p-%p\n", start, start + ic->slot_size);
     memcpy(slot_start, buf, ic->getSlotSize());
 
+    ic_entry->gc_references = std::move(gc_references);
+
     ic->times_rewritten++;
 
     if (ic->times_rewritten == IC_MEGAMORPHIC_THRESHOLD) {
@@ -187,6 +191,25 @@ ICSlotInfo* ICInfo::pickEntryForRewrite(const char* debug_name) {
     return NULL;
 }
 
+// Keep track of all ICInfo(s) that we create because they contain pointers to Pyston heap objects
+// that we have written into the generated code and we may need to scan those.
+static llvm::DenseSet<ICInfo*> ics_list;
+static llvm::DenseMap<void*, ICInfo*> ics_by_return_addr;
+
+void registerGCTrackedICInfo(ICInfo* ic) {
+#if MOVING_GC
+    assert(ics_list.count(ic) == 0);
+    ics_list.insert(ic);
+#endif
+}
+
+void deregisterGCTrackedICInfo(ICInfo* ic) {
+#if MOVING_GC
+    assert(ics_list.count(ic) == 1);
+    ics_list.erase(ic);
+#endif
+}
+
 ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
                int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet _live_outs,
                assembler::GenericRegister return_register, TypeRecorder* type_recorder)
@@ -207,9 +230,18 @@ ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, S
     for (int i = 0; i < num_slots; i++) {
         slots.emplace_back(this, i);
     }
+
+#if MOVING_GC
+    assert(ics_list.count(this) == 0);
+#endif
+}
+
+ICInfo::~ICInfo() {
+#if MOVING_GC
+    assert(ics_list.count(this) == 0);
+#endif
 }
 
-static llvm::DenseMap<void*, ICInfo*> ics_by_return_addr;
 std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr,
                                                    uint8_t* continue_addr, uint8_t* slowpath_rtn_addr,
                                                    const ICSetupInfo* ic, StackInfo stack_info, LiveOutSet live_outs) {
@@ -251,14 +283,19 @@ std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t*
     ICInfo* icinfo = new ICInfo(start_addr, slowpath_rtn_addr, continue_addr, stack_info, ic->num_slots, ic->slot_size,
                                 ic->getCallingConvention(), std::move(live_outs), return_register, ic->type_recorder);
 
+    assert(!ics_by_return_addr.count(slowpath_rtn_addr));
     ics_by_return_addr[slowpath_rtn_addr] = icinfo;
 
+    registerGCTrackedICInfo(icinfo);
+
     return std::unique_ptr<ICInfo>(icinfo);
 }
 
 void deregisterCompiledPatchpoint(ICInfo* ic) {
     assert(ics_by_return_addr.count(ic->slowpath_rtn_addr));
     ics_by_return_addr.erase(ic->slowpath_rtn_addr);
+
+    deregisterGCTrackedICInfo(ic);
 }
 
 ICInfo* getICInfo(void* rtn_addr) {
@@ -305,4 +342,12 @@ bool ICInfo::shouldAttempt() {
 bool ICInfo::isMegamorphic() {
     return times_rewritten >= IC_MEGAMORPHIC_THRESHOLD;
 }
+
+void ICInfo::visitGCReferences(gc::GCVisitor* v) {
+    for (const auto& p : ics_list) {
+        for (auto& slot : p->slots) {
+            v->visitNonRelocatableRange(&slot.gc_references[0], &slot.gc_references[slot.gc_references.size()]);
+        }
+    }
+}
 }

src/asm_writing/icinfo.h

@@ -27,6 +27,10 @@
 
 namespace pyston {
 
+namespace gc {
+class GCVisitor;
+}
+
 class TypeRecorder;
 
 class ICInfo;
@@ -43,6 +47,8 @@ public:
     int idx;        // the index inside the ic
     int num_inside; // the number of stack frames that are currently inside this slot
 
+    std::vector<void*> gc_references;
+
     void clear();
 };
 
@@ -85,7 +91,7 @@ public:
     void gc_visit(gc::GCVisitor* visitor);
 
     void addDependenceOn(ICInvalidator&);
-    void commit(CommitHook* hook);
+    void commit(CommitHook* hook, std::vector<void*> gc_references);
     void abort();
 
     const ICInfo* getICInfo() { return ic; }
@@ -124,6 +130,7 @@ public:
     ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int num_slots,
            int slot_size, llvm::CallingConv::ID calling_conv, LiveOutSet live_outs,
            assembler::GenericRegister return_register, TypeRecorder* type_recorder);
+    ~ICInfo();
     void* const start_addr, *const slowpath_rtn_addr, *const continue_addr;
 
     int getSlotSize() { return slot_size; }
@@ -142,8 +149,12 @@ public:
     int percentBackedoff() const { return retry_backoff; }
 
     friend class ICSlotRewrite;
+    static void visitGCReferences(gc::GCVisitor* visitor);
 };
 
+void registerGCTrackedICInfo(ICInfo* ic);
+void deregisterGCTrackedICInfo(ICInfo* ic);
+
 class ICSetupInfo;
 struct CompiledFunction;
 std::unique_ptr<ICInfo> registerCompiledPatchpoint(uint8_t* start_addr, uint8_t* slowpath_start_addr,

src/asm_writing/rewriter.cpp

@@ -19,6 +19,7 @@
 #include "codegen/unwinding.h"
 #include "core/common.h"
 #include "core/stats.h"
+#include "gc/heap.h"
 #include "runtime/types.h"
 
 namespace pyston {
@@ -350,6 +351,7 @@ void RewriterVar::addAttrGuard(int offset, uint64_t val, bool negate) {
 
     if (!attr_guards.insert(std::make_tuple(offset, val, negate)).second)
         return; // duplicate guard detected
+
     RewriterVar* val_var = rewriter->loadConst(val);
     rewriter->addAction([=]() { rewriter->_addAttrGuard(this, offset, val_var, negate); }, { this, val_var },
                         ActionType::GUARD);
@@ -717,6 +719,15 @@ void Rewriter::_trap() {
 RewriterVar* Rewriter::loadConst(int64_t val, Location dest) {
     STAT_TIMER(t0, "us_timer_rewriter", 10);
 
+#if MOVING_GC
+    // Moving GCs need to know all existing references in the program, including those that end
+    // up in compiled code.
+    auto al = gc::global_heap.getAllocationFromInteriorPointer((void*)val);
+    if (al) {
+        gc_references.push_back(al->user_data);
+    }
+#endif
+
     for (auto& p : const_loader.consts) {
         if (p.first != val)
             continue;
@@ -1328,7 +1339,7 @@ void Rewriter::commit() {
     }
 #endif
 
-    rewrite->commit(this);
+    rewrite->commit(this, std::move(gc_references));
 
     if (assembler->hasFailed()) {
         on_assemblyfail();
@@ -1378,6 +1389,12 @@ void Rewriter::addDependenceOn(ICInvalidator& invalidator) {
 
 void Rewriter::gc_visit(GCVisitor* visitor) {
     rewrite->gc_visit(visitor);
+
+    // A GC could happen during a rewrite, so we need to scan the list of references
+    // both as it is being built and after the rewrite commits.
+    for (auto& reference : gc_references) {
+        visitor->visitNonRelocatable(reference);
+    }
 }
 
 Location Rewriter::allocScratch() {

src/asm_writing/rewriter.h

@@ -442,6 +442,7 @@ protected:
     }
     bool added_changing_action;
     bool marked_inside_ic;
+    std::vector<void*> gc_references;
 
     bool done_guarding;
     bool isDoneGuarding() {

src/codegen/baseline_jit.cpp

@@ -624,6 +624,15 @@ int JitFragmentWriter::finishCompilation() {
 
     void* next_fragment_start = (uint8_t*)block->code + assembler->bytesWritten();
     code_block.fragmentFinished(assembler->bytesWritten(), num_bytes_overlapping, next_fragment_start);
+
+#if MOVING_GC
+    // If JitFragmentWriter is destroyed, we don't necessarily want the ICInfo to be destroyed also,
+    // because it may contain a list of references to pointers in generated code that still exists
+    // and we need to keep those around.
+    // TODO: When should these ICInfo be freed?
+    registerGCTrackedICInfo(ic_info.release());
+#endif
+
     return num_bytes_exit;
 }
 

src/codegen/irgen.cpp

@@ -1042,6 +1042,7 @@ CompiledFunction* doCompile(CLFunction* clfunc, SourceInfo* source, ParamNames*
     }
 
     CompiledFunction* cf = new CompiledFunction(NULL, spec, NULL, effort, exception_style, entry_descriptor);
+    setPointersInCodeStorage(&cf->pointers_in_code);
 
     // Make sure that the instruction memory keeps the module object alive.
     // TODO: implement this for real

src/codegen/irgen/hooks.cpp

@@ -753,6 +753,7 @@ void CompiledFunction::speculationFailed() {
     }
 }
 
+std::unordered_set<CompiledFunction*> all_compiled_functions;
 CompiledFunction::CompiledFunction(llvm::Function* func, FunctionSpecialization* spec, void* code, EffortLevel effort,
                                    ExceptionStyle exception_style, const OSREntryDescriptor* entry_descriptor)
     : clfunc(NULL),
@@ -766,6 +767,26 @@ CompiledFunction::CompiledFunction(llvm::Function* func, FunctionSpecialization*
       times_speculation_failed(0),
       location_map(nullptr) {
     assert((spec != NULL) + (entry_descriptor != NULL) == 1);
+
+#if MOVING_GC
+    assert(all_compiled_functions.count(this) == 0);
+    all_compiled_functions.insert(this);
+#endif
+}
+
+#if MOVING_GC
+CompiledFunction::~CompiledFunction() {
+    assert(all_compiled_functions.count(this) == 1);
+    all_compiled_functions.erase(this);
+}
+#endif
+
+void CompiledFunction::visitAllCompiledFunctions(GCVisitor* visitor) {
+    for (CompiledFunction* cf : all_compiled_functions) {
+        for (const void* ptr : cf->pointers_in_code) {
+            visitor->visitNonRelocatable(const_cast<void*>(ptr));
+        }
+    }
 }
 
 ConcreteCompilerType* CompiledFunction::getReturnType() {

src/codegen/irgen/util.cpp

@@ -27,6 +27,7 @@
 #include "codegen/codegen.h"
 #include "codegen/patchpoints.h"
 #include "core/common.h"
+#include "gc/gc.h"
 #include "runtime/types.h"
 
 namespace pyston {
@@ -99,10 +100,18 @@ llvm::Constant* getStringConstantPtr(llvm::StringRef str) {
 
 static llvm::StringMap<const void*> relocatable_syms;
 
+// Pointer to a vector where we want to keep track of all the pointers written directly into
+// the compiled code, which the GC needs to be aware of.
+std::vector<const void*>* pointers_in_code;
+
 void clearRelocatableSymsMap() {
     relocatable_syms.clear();
 }
 
+void setPointersInCodeStorage(std::vector<const void*>* v) {
+    pointers_in_code = v;
+}
+
 const void* getValueOfRelocatableSym(const std::string& str) {
     auto it = relocatable_syms.find(str);
     if (it != relocatable_syms.end())
@@ -129,6 +138,13 @@ llvm::Constant* embedRelocatablePtr(const void* addr, llvm::Type* type, llvm::St
 
     relocatable_syms[name] = addr;
 
+#if MOVING_GC
+    gc::GCAllocation* al = gc::global_heap.getAllocationFromInteriorPointer(const_cast<void*>(addr));
+    if (al) {
+        pointers_in_code->push_back(al->user_data);
+    }
+#endif
+
     llvm::Type* var_type = type->getPointerElementType();
     return new llvm::GlobalVariable(*g.cur_module, var_type, true, llvm::GlobalVariable::ExternalLinkage, 0, name);
 }
@@ -137,6 +153,14 @@ llvm::Constant* embedConstantPtr(const void* addr, llvm::Type* type) {
     assert(type);
     llvm::Constant* int_val = llvm::ConstantInt::get(g.i64, reinterpret_cast<uintptr_t>(addr), false);
     llvm::Constant* ptr_val = llvm::ConstantExpr::getIntToPtr(int_val, type);
+
+#if MOVING_GC
+    gc::GCAllocation* al = gc::global_heap.getAllocationFromInteriorPointer(const_cast<void*>(addr));
+    if (al) {
+        pointers_in_code->push_back(al->user_data);
+    }
+#endif
+
     return ptr_val;
 }
 

src/codegen/irgen/util.h

@@ -17,6 +17,7 @@
 
 #include <string>
 
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 
 namespace llvm {
@@ -25,6 +26,10 @@ class Function;
 class Type;
 }
 
+namespace gc {
+class GCVisitor;
+}
+
 namespace pyston {
 
 llvm::Constant* embedRelocatablePtr(const void* addr, llvm::Type*, llvm::StringRef shared_name = llvm::StringRef());
@@ -35,8 +40,11 @@ llvm::Constant* getConstantInt(int64_t val, llvm::Type*);
 llvm::Constant* getNullPtr(llvm::Type* t);
 
 void clearRelocatableSymsMap();
+void setPointersInCodeStorage(std::vector<const void*>* v);
 const void* getValueOfRelocatableSym(const std::string& str);
 
+void visitRelocatableSymsMap(gc::GCVisitor* visitor);
+
 void dumpPrettyIR(llvm::Function* f);
 }
 

src/core/types.h

@@ -262,6 +262,9 @@ public:
     FunctionSpecialization* spec;
     const OSREntryDescriptor* entry_descriptor;
 
+    // Pointers that were written directly into the code, which the GC should be aware of.
+    std::vector<const void*> pointers_in_code;
+
     union {
         Box* (*call)(Box*, Box*, Box*, Box**);
         Box* (*closure_call)(BoxedClosure*, Box*, Box*, Box*, Box**);
@@ -298,6 +301,8 @@ public:
 
     // Call this when a speculation inside this version failed
     void speculationFailed();
+
+    static void visitAllCompiledFunctions(GCVisitor* visitor);
 };
 
 typedef int FutureFlags;

src/gc/collector.cpp

@@ -18,8 +18,10 @@
 #include <cstdio>
 #include <cstdlib>
 
+#include "asm_writing/icinfo.h"
 #include "codegen/ast_interpreter.h"
 #include "codegen/codegen.h"
+#include "codegen/irgen/util.h"
 #include "core/common.h"
 #include "core/threading.h"
 #include "core/types.h"
@@ -429,6 +431,12 @@ static void markRoots(GCVisitor& visitor) {
     for (auto weakref : weakrefs_needing_callback_list) {
         visitor.visit(weakref);
     }
+
+    GC_TRACE_LOG("Looking at generated code pointers\n");
+#if MOVING_GC
+    ICInfo::visitGCReferences(&visitor);
+    CompiledFunction::visitAllCompiledFunctions(&visitor);
+#endif
 }
 
 static void finalizationOrderingFindReachable(Box* obj) {

src/gc/gc.h

@@ -86,6 +86,14 @@ public:
     virtual void visitRedundantRange(void** start, void** end) {}
     virtual void visitPotentialRedundant(void* p) {}
     virtual void visitPotentialRangeRedundant(void* const* start, void* const* end) {}
+
+    // Visit pointers to objects that we know cannot be moved.
+    // This is often used to scan a pointer that's a copy of a pointer stored in a place that
+    // we cannot easily scanned (like generated code).
+    // This default to visitPotential for now (which also cannot be moved) but we may want to
+    // change that later for performance.
+    void visitNonRelocatable(void* p) { visitPotential(p); }
+    void visitNonRelocatableRange(void** start, void** end) { visitPotentialRange(start, end); }
 };
 
 enum class GCKind : uint8_t {