Merge pull request #494 from kmod/analysis_mem3

Reduce memory by not caching analysis passes

Makefile

@@ -779,11 +779,16 @@ endef
 PASS_SRCS := codegen/opt/aa.cpp
 PASS_OBJS := $(PASS_SRCS:.cpp=.standalone.o)
 
+ifneq ($(USE_CMAKE),1)
 $(call make_compile_config,,$(CXXFLAGS_DBG))
 $(call make_compile_config,.release,$(CXXFLAGS_RELEASE))
 $(call make_compile_config,.grwl,$(CXXFLAGS_RELEASE) -DTHREADING_USE_GRWL=1 -DTHREADING_USE_GIL=0 -UBINARY_SUFFIX -DBINARY_SUFFIX=_grwl)
 $(call make_compile_config,.grwl_dbg,$(CXXFLAGS_DBG) -DTHREADING_USE_GRWL=1 -DTHREADING_USE_GIL=0 -UBINARY_SUFFIX -DBINARY_SUFFIX=_grwl_dbg -UBINARY_STRIPPED_SUFFIX -DBINARY_STRIPPED_SUFFIX=)
 $(call make_compile_config,.nosync,$(CXXFLAGS_RELEASE) -DTHREADING_USE_GRWL=0 -DTHREADING_USE_GIL=0 -UBINARY_SUFFIX -DBINARY_SUFFIX=_nosync)
+else
+%.o: %.cpp $(CMAKE_SETUP_DBG)
+	$(NINJA) -C $(HOME)/pyston-build-dbg src/CMakeFiles/PYSTON_OBJECTS.dir/$(patsubst src/%.o,%.cpp.o,$@) $(NINJAFLAGS)
+endif
 
 $(UNITTEST_SRCS:.cpp=.o): CXXFLAGS += -isystem $(GTEST_DIR)/include
 

src/analysis/function_analysis.cpp

@@ -76,14 +76,21 @@ private:
         }
     }
 
+    Status::Usage getStatusFirst(InternedString name) const {
+        auto it = statuses.find(name);
+        if (it == statuses.end())
+            return Status::NONE;
+        return it->second.first;
+    }
+
 public:
     LivenessBBVisitor(LivenessAnalysis* analysis) : analysis(analysis) {}
 
-    bool firstIsUse(InternedString name) { return statuses[name].first == Status::USED; }
+    bool firstIsUse(InternedString name) const { return getStatusFirst(name) == Status::USED; }
 
-    bool firstIsDef(InternedString name) { return statuses[name].first == Status::DEFINED; }
+    bool firstIsDef(InternedString name) const { return getStatusFirst(name) == Status::DEFINED; }
 
-    bool isKilledAt(AST_Name* node, bool is_live_at_end) {
+    bool isKilledAt(AST_Name* node, bool is_live_at_end) const {
         if (kills.count(node))
             return true;
 
@@ -163,6 +170,9 @@ LivenessAnalysis::LivenessAnalysis(CFG* cfg) : cfg(cfg) {
     us_liveness.log(_t.end());
 }
 
+LivenessAnalysis::~LivenessAnalysis() {
+}
+
 bool LivenessAnalysis::isKill(AST_Name* node, CFGBlock* parent_block) {
     if (node->id.str()[0] != '#')
         return false;
@@ -487,11 +497,18 @@ bool PhiAnalysis::isPotentiallyUndefinedAt(InternedString name, CFGBlock* block)
     return definedness.defined_at_beginning[block][name] != DefinednessAnalysis::Defined;
 }
 
-LivenessAnalysis* computeLivenessInfo(CFG* cfg) {
-    return new LivenessAnalysis(cfg);
+std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG* cfg) {
+    static StatCounter counter("num_liveness_analysis");
+    counter.log();
+
+    return std::unique_ptr<LivenessAnalysis>(new LivenessAnalysis(cfg));
 }
 
-PhiAnalysis* computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnalysis* liveness, ScopeInfo* scope_info) {
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnalysis* liveness,
+                                                 ScopeInfo* scope_info) {
+    static StatCounter counter("num_phi_analysis");
+    counter.log();
+
     llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel> initial_map;
 
     for (auto e : args.args)
@@ -501,11 +518,15 @@ PhiAnalysis* computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnaly
     if (args.kwarg.size())
         initial_map[scope_info->internString(args.kwarg)] = DefinednessAnalysis::Defined;
 
-    return new PhiAnalysis(std::move(initial_map), cfg->getStartingBlock(), false, liveness, scope_info);
+    return std::unique_ptr<PhiAnalysis>(
+        new PhiAnalysis(std::move(initial_map), cfg->getStartingBlock(), false, liveness, scope_info));
 }
 
-PhiAnalysis* computeRequiredPhis(const OSREntryDescriptor* entry_descriptor, LivenessAnalysis* liveness,
-                                 ScopeInfo* scope_info) {
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry_descriptor, LivenessAnalysis* liveness,
+                                                 ScopeInfo* scope_info) {
+    static StatCounter counter("num_phi_analysis");
+    counter.log();
+
     llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel> initial_map;
 
     llvm::StringSet<> potentially_undefined;
@@ -524,6 +545,7 @@ PhiAnalysis* computeRequiredPhis(const OSREntryDescriptor* entry_descriptor, Liv
             initial_map[p.first] = DefinednessAnalysis::Defined;
     }
 
-    return new PhiAnalysis(std::move(initial_map), entry_descriptor->backedge->target, true, liveness, scope_info);
+    return std::unique_ptr<PhiAnalysis>(
+        new PhiAnalysis(std::move(initial_map), entry_descriptor->backedge->target, true, liveness, scope_info));
 }
 }

src/analysis/function_analysis.h

@@ -45,6 +45,7 @@ private:
 
 public:
     LivenessAnalysis(CFG* cfg);
+    ~LivenessAnalysis();
 
     // we don't keep track of node->parent_block relationships, so you have to pass both:
     bool isKill(AST_Name* node, CFGBlock* parent_block);
@@ -105,9 +106,9 @@ public:
     bool isPotentiallyUndefinedAt(InternedString name, CFGBlock* block);
 };
 
-LivenessAnalysis* computeLivenessInfo(CFG*);
-PhiAnalysis* computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*, ScopeInfo* scope_info);
-PhiAnalysis* computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*, ScopeInfo* scope_info);
+std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG*);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*, ScopeInfo* scope_info);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*, ScopeInfo* scope_info);
 }
 
 #endif

src/codegen/ast_interpreter.cpp

@@ -83,7 +83,6 @@ private:
     Value doBinOp(Box* left, Box* right, int op, BinExpType exp_type);
     void doStore(AST_expr* node, Value value);
     void doStore(InternedString name, Value value);
-    void eraseDeadSymbols();
 
     Value visit_assert(AST_Assert* node);
     Value visit_assign(AST_Assign* node);
@@ -321,31 +320,6 @@ Value ASTInterpreter::execute(ASTInterpreter& interpreter, CFGBlock* start_block
     return v;
 }
 
-void ASTInterpreter::eraseDeadSymbols() {
-    if (source_info->liveness == NULL)
-        source_info->liveness = computeLivenessInfo(source_info->cfg);
-
-    if (this->phis == NULL) {
-        PhiAnalysis*& phis = source_info->phis[/* entry_descriptor = */ NULL];
-        if (!phis)
-            phis = computeRequiredPhis(compiled_func->clfunc->param_names, source_info->cfg, source_info->liveness,
-                                       scope_info);
-        this->phis = phis;
-    }
-
-    std::vector<InternedString> dead_symbols;
-    for (auto& it : sym_table) {
-        if (!source_info->liveness->isLiveAtEnd(it.first, current_block)) {
-            dead_symbols.push_back(it.first);
-        } else if (phis->isRequiredAfter(it.first, current_block)) {
-            assert(scope_info->getScopeTypeOfName(it.first) != ScopeInfo::VarScopeType::GLOBAL);
-        } else {
-        }
-    }
-    for (auto&& dead : dead_symbols)
-        sym_table.erase(dead);
-}
-
 Value ASTInterpreter::doBinOp(Box* left, Box* right, int op, BinExpType exp_type) {
     if (op == AST_TYPE::Div && (source_info->parent_module->future_flags & FF_DIVISION)) {
         op = AST_TYPE::TrueDiv;
@@ -463,7 +437,26 @@ Value ASTInterpreter::visit_jump(AST_Jump* node) {
     if (ENABLE_OSR && backedge && (globals->cls == module_cls)) {
         bool can_osr = !FORCE_INTERPRETER && (globals->cls == module_cls);
         if (can_osr && edgecount++ == OSR_THRESHOLD_INTERPRETER) {
-            eraseDeadSymbols();
+            static StatCounter ast_osrs("num_ast_osrs");
+            ast_osrs.log();
+
+            // TODO: we will immediately want the liveness info again in the jit, we should pass
+            // it through.
+            std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(source_info->cfg);
+            std::unique_ptr<PhiAnalysis> phis
+                = computeRequiredPhis(compiled_func->clfunc->param_names, source_info->cfg, liveness.get(), scope_info);
+
+            std::vector<InternedString> dead_symbols;
+            for (auto& it : sym_table) {
+                if (!liveness->isLiveAtEnd(it.first, current_block)) {
+                    dead_symbols.push_back(it.first);
+                } else if (phis->isRequiredAfter(it.first, current_block)) {
+                    assert(scope_info->getScopeTypeOfName(it.first) != ScopeInfo::VarScopeType::GLOBAL);
+                } else {
+                }
+            }
+            for (auto&& dead : dead_symbols)
+                sym_table.erase(dead);
 
             const OSREntryDescriptor* found_entry = nullptr;
             for (auto& p : compiled_func->clfunc->osr_versions) {
@@ -479,7 +472,7 @@ Value ASTInterpreter::visit_jump(AST_Jump* node) {
 
             for (auto& name : phis->definedness.getDefinedNamesAtEnd(current_block)) {
                 auto it = sym_table.find(name);
-                if (!source_info->liveness->isLiveAtEnd(name, current_block))
+                if (!liveness->isLiveAtEnd(name, current_block))
                     continue;
 
                 if (phis->isPotentiallyUndefinedAfter(name, current_block)) {
@@ -495,10 +488,14 @@ Value ASTInterpreter::visit_jump(AST_Jump* node) {
                 }
             }
 
+            // Manually free these here, since we might not return from this scope for a long time.
+            liveness.reset(nullptr);
+            phis.reset(nullptr);
+
             // LLVM has a limit on the number of operands a machine instruction can have (~255),
             // in order to not hit the limit with the patchpoints cancel OSR when we have a high number of symbols.
             if (sorted_symbol_table.size() > 225) {
-                static StatCounter times_osr_cancel("num_osr_cancel_to_many_syms");
+                static StatCounter times_osr_cancel("num_osr_cancel_too_many_syms");
                 times_osr_cancel.log();
                 next_block = node->target;
                 return Value();

src/codegen/codegen.cpp

@@ -35,8 +35,7 @@ namespace pyston {
 DS_DEFINE_RWLOCK(codegen_rwlock);
 
 SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, std::vector<AST_stmt*> body, std::string fn)
-    : parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), fn(std::move(fn)),
-      body(std::move(body)) {
+    : parent_module(m), scoping(scoping), ast(ast), cfg(NULL), fn(std::move(fn)), body(std::move(body)) {
     assert(this->fn.size());
 
     switch (ast->type) {

src/codegen/irgen.cpp

@@ -342,7 +342,7 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
     CompiledFunction* cf = irstate->getCurFunction();
     ConcreteCompilerType* rtn_type = irstate->getReturnType();
     // llvm::MDNode* func_info = irstate->getFuncDbgInfo();
-    PhiAnalysis* phi_analysis = source->phis[entry_descriptor];
+    PhiAnalysis* phi_analysis = irstate->getPhis();
     assert(phi_analysis);
 
     if (entry_descriptor != NULL)
@@ -1055,7 +1055,15 @@ CompiledFunction* doCompile(SourceInfo* source, ParamNames* param_names, const O
         computeBlockSetClosure(blocks);
     }
 
-    IRGenState irstate(cf, source, source->phis[entry_descriptor], param_names, getGCBuilder(), dbg_funcinfo);
+    std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(source->cfg);
+    std::unique_ptr<PhiAnalysis> phis;
+
+    if (entry_descriptor)
+        phis = computeRequiredPhis(entry_descriptor, liveness.get(), source->getScopeInfo());
+    else
+        phis = computeRequiredPhis(*param_names, source->cfg, liveness.get(), source->getScopeInfo());
+
+    IRGenState irstate(cf, source, std::move(liveness), std::move(phis), param_names, getGCBuilder(), dbg_funcinfo);
 
     emitBBs(&irstate, types, entry_descriptor, blocks);
 

src/codegen/irgen/hooks.cpp

@@ -228,19 +228,6 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
         source->cfg = computeCFG(source, source->body);
     }
 
-    if (effort != EffortLevel::INTERPRETED) {
-        if (source->liveness == NULL)
-            source->liveness = computeLivenessInfo(source->cfg);
-
-        PhiAnalysis*& phis = source->phis[entry_descriptor];
-        if (!phis) {
-            if (entry_descriptor)
-                phis = computeRequiredPhis(entry_descriptor, source->liveness, source->getScopeInfo());
-            else
-                phis = computeRequiredPhis(f->param_names, source->cfg, source->liveness, source->getScopeInfo());
-        }
-    }
-
 
 
     CompiledFunction* cf = 0;

src/codegen/irgen/irgenerator.cpp

@@ -42,6 +42,19 @@ extern "C" void dumpLLVM(llvm::Value* v) {
     v->dump();
 }
 
+IRGenState::IRGenState(CompiledFunction* cf, SourceInfo* source_info, std::unique_ptr<LivenessAnalysis> liveness,
+                       std::unique_ptr<PhiAnalysis> phis, ParamNames* param_names, GCBuilder* gc,
+                       llvm::MDNode* func_dbg_info)
+    : cf(cf), source_info(source_info), liveness(std::move(liveness)), phis(std::move(phis)), param_names(param_names),
+      gc(gc), func_dbg_info(func_dbg_info), scratch_space(NULL), frame_info(NULL), frame_info_arg(NULL),
+      scratch_size(0) {
+    assert(cf->func);
+    assert(!cf->clfunc); // in this case don't need to pass in sourceinfo
+}
+
+IRGenState::~IRGenState() {
+}
+
 llvm::Value* IRGenState::getScratchSpace(int min_bytes) {
     llvm::BasicBlock& entry_block = getLLVMFunction()->getEntryBlock();
 
@@ -944,7 +957,7 @@ private:
     CompilerVariable* evalName(AST_Name* node, UnwindInfo unw_info) {
         auto scope_info = irstate->getScopeInfo();
 
-        bool is_kill = irstate->getSourceInfo()->liveness->isKill(node, myblock);
+        bool is_kill = irstate->getLiveness()->isKill(node, myblock);
         assert(!is_kill || node->id.str()[0] == '#');
 
         ScopeInfo::VarScopeType vst = scope_info->getScopeTypeOfName(node->id);
@@ -2271,7 +2284,7 @@ private:
             // ASSERT(p.first[0] != '!' || isIsDefinedName(p.first), "left a fake variable in the real
             // symbol table? '%s'", p.first.c_str());
 
-            if (!source->liveness->isLiveAtEnd(p.first, myblock)) {
+            if (!irstate->getLiveness()->isLiveAtEnd(p.first, myblock)) {
                 // printf("%s dead at end of %d; grabbed = %d, %d vrefs\n", p.first.c_str(), myblock->idx,
                 //        p.second->isGrabbed(), p.second->getVrefs());
 

src/codegen/irgen/irgenerator.h

@@ -56,7 +56,8 @@ class IRGenState {
 private:
     CompiledFunction* cf;
     SourceInfo* source_info;
-    PhiAnalysis* phis;
+    std::unique_ptr<LivenessAnalysis> liveness;
+    std::unique_ptr<PhiAnalysis> phis;
     ParamNames* param_names;
     GCBuilder* gc;
     llvm::MDNode* func_dbg_info;
@@ -69,13 +70,9 @@ private:
 
 
 public:
-    IRGenState(CompiledFunction* cf, SourceInfo* source_info, PhiAnalysis* phis, ParamNames* param_names, GCBuilder* gc,
-               llvm::MDNode* func_dbg_info)
-        : cf(cf), source_info(source_info), phis(phis), param_names(param_names), gc(gc), func_dbg_info(func_dbg_info),
-          scratch_space(NULL), frame_info(NULL), frame_info_arg(NULL), scratch_size(0) {
-        assert(cf->func);
-        assert(!cf->clfunc); // in this case don't need to pass in sourceinfo
-    }
+    IRGenState(CompiledFunction* cf, SourceInfo* source_info, std::unique_ptr<LivenessAnalysis> liveness,
+               std::unique_ptr<PhiAnalysis> phis, ParamNames* param_names, GCBuilder* gc, llvm::MDNode* func_dbg_info);
+    ~IRGenState();
 
     CompiledFunction* getCurFunction() { return cf; }
 
@@ -93,7 +90,8 @@ public:
 
     SourceInfo* getSourceInfo() { return source_info; }
 
-    PhiAnalysis* getPhis() { return phis; }
+    LivenessAnalysis* getLiveness() { return liveness.get(); }
+    PhiAnalysis* getPhis() { return phis.get(); }
 
     ScopeInfo* getScopeInfo();
     ScopeInfo* getScopeInfoForNode(AST* node);

src/core/stats.cpp

@@ -17,6 +17,7 @@
 #include <algorithm>
 
 #include "core/thread_utils.h"
+#include "gc/heap.h"
 
 namespace pyston {
 
@@ -55,7 +56,11 @@ void Stats::dump(bool includeZeros) {
     if (!Stats::enabled)
         return;
 
-    printf("Stats:\n");
+    fprintf(stderr, "Stats:\n");
+
+    gc::dumpHeapStatistics(0);
+
+    fprintf(stderr, "Counters:\n");
 
     std::vector<std::pair<std::string, int>> pairs;
     for (const auto& p : *names) {
@@ -66,8 +71,10 @@ void Stats::dump(bool includeZeros) {
 
     for (int i = 0; i < pairs.size(); i++) {
         if (includeZeros || (*counts)[pairs[i].second] > 0)
-            printf("%s: %ld\n", pairs[i].first.c_str(), (*counts)[pairs[i].second]);
+            fprintf(stderr, "%s: %ld\n", pairs[i].first.c_str(), (*counts)[pairs[i].second]);
     }
+
+    fprintf(stderr, "(End of stats)\n");
 }
 
 void Stats::endOfInit() {

src/core/types.h

@@ -243,8 +243,6 @@ public:
     ScopingAnalysis* scoping;
     AST* ast;
     CFG* cfg;
-    LivenessAnalysis* liveness;
-    std::unordered_map<const OSREntryDescriptor*, PhiAnalysis*> phis;
     bool is_generator;
     std::string fn; // equivalent of code.co_filename
 

src/gc/heap.cpp

@@ -185,16 +185,32 @@ struct HeapStatistics {
 
         void print(const char* name) const {
             if (nbytes > (1 << 20))
-                printf("%s: %ld allocations for %.1f MB\n", name, nallocs, nbytes * 1.0 / (1 << 20));
+                fprintf(stderr, "%s: %ld allocations for %.1f MB\n", name, nallocs, nbytes * 1.0 / (1 << 20));
             else if (nbytes > (1 << 10))
-                printf("%s: %ld allocations for %.1f KB\n", name, nallocs, nbytes * 1.0 / (1 << 10));
+                fprintf(stderr, "%s: %ld allocations for %.1f KB\n", name, nallocs, nbytes * 1.0 / (1 << 10));
             else
-                printf("%s: %ld allocations for %ld bytes\n", name, nallocs, nbytes);
+                fprintf(stderr, "%s: %ld allocations for %ld bytes\n", name, nallocs, nbytes);
         }
     };
+
+    bool collect_cls_stats, collect_hcls_stats;
+
+    // For use if collect_cls_stats == true:
     std::unordered_map<BoxedClass*, TypeStats> by_cls;
-    TypeStats conservative, untracked, hcls;
+
+    // For use if collect_hcls_stats == true:
+    std::unordered_map<HiddenClass*, int> hcls_uses;
+#define HCLS_ATTRS_STAT_MAX 20
+    int num_hcls_by_attrs[HCLS_ATTRS_STAT_MAX + 1];
+    int num_hcls_by_attrs_exceed;
+
+    TypeStats python, conservative, untracked, hcls, precise;
     TypeStats total;
+
+    HeapStatistics(bool collect_cls_stats, bool collect_hcls_stats)
+        : collect_cls_stats(collect_cls_stats), collect_hcls_stats(collect_hcls_stats) {
+        memset(num_hcls_by_attrs, 0, sizeof(num_hcls_by_attrs));
+    }
 };
 
 void addStatistic(HeapStatistics* stats, GCAllocation* al, int nbytes) {
@@ -202,11 +218,24 @@ void addStatistic(HeapStatistics* stats, GCAllocation* al, int nbytes) {
     stats->total.nbytes += nbytes;
 
     if (al->kind_id == GCKind::PYTHON) {
-        Box* b = (Box*)al->user_data;
-        auto& t = stats->by_cls[b->cls];
+        stats->python.nallocs++;
+        stats->python.nbytes += nbytes;
 
-        t.nallocs++;
-        t.nbytes += nbytes;
+        if (stats->collect_cls_stats) {
+            Box* b = (Box*)al->user_data;
+            auto& t = stats->by_cls[b->cls];
+
+            t.nallocs++;
+            t.nbytes += nbytes;
+        }
+
+        if (stats->collect_hcls_stats) {
+            Box* b = (Box*)al->user_data;
+            if (b->cls->instancesHaveHCAttrs()) {
+                HCAttrs* attrs = b->getHCAttrsPtr();
+                stats->hcls_uses[attrs->hcls]++;
+            }
+        }
     } else if (al->kind_id == GCKind::CONSERVATIVE) {
         stats->conservative.nallocs++;
         stats->conservative.nbytes += nbytes;
@@ -216,6 +245,18 @@ void addStatistic(HeapStatistics* stats, GCAllocation* al, int nbytes) {
     } else if (al->kind_id == GCKind::HIDDEN_CLASS) {
         stats->hcls.nallocs++;
         stats->hcls.nbytes += nbytes;
+
+        if (stats->collect_hcls_stats) {
+            HiddenClass* hcls = (HiddenClass*)al->user_data;
+            int numattrs = hcls->attributeArraySize();
+            if (numattrs <= HCLS_ATTRS_STAT_MAX)
+                stats->num_hcls_by_attrs[numattrs]++;
+            else
+                stats->num_hcls_by_attrs_exceed++;
+        }
+    } else if (al->kind_id == GCKind::PRECISE) {
+        stats->precise.nallocs++;
+        stats->precise.nbytes += nbytes;
     } else {
         RELEASE_ASSERT(0, "%d", (int)al->kind_id);
     }
@@ -223,27 +264,49 @@ void addStatistic(HeapStatistics* stats, GCAllocation* al, int nbytes) {
 
 
 
-void Heap::dumpHeapStatistics() {
+void Heap::dumpHeapStatistics(int level) {
+    bool collect_cls_stats = (level >= 1);
+    bool collect_hcls_stats = (level >= 1);
+
     threading::GLPromoteRegion _lock;
 
-    HeapStatistics stats;
+    fprintf(stderr, "\nCollecting heap stats for pid %d...\n", getpid());
+
+    HeapStatistics stats(collect_cls_stats, collect_hcls_stats);
 
     small_arena.getStatistics(&stats);
     large_arena.getStatistics(&stats);
     huge_arena.getStatistics(&stats);
 
+    stats.python.print("python");
     stats.conservative.print("conservative");
     stats.untracked.print("untracked");
     stats.hcls.print("hcls");
-    for (const auto& p : stats.by_cls) {
-        p.second.print(getFullNameOfClass(p.first).c_str());
+    stats.precise.print("precise");
+
+    if (collect_cls_stats) {
+        for (const auto& p : stats.by_cls) {
+            p.second.print(getFullNameOfClass(p.first).c_str());
+        }
     }
+
     stats.total.print("Total");
-    printf("\n");
+
+    if (collect_hcls_stats) {
+        fprintf(stderr, "%ld hidden classes currently alive\n", stats.hcls.nallocs);
+        fprintf(stderr, "%ld have at least one Box that uses them\n", stats.hcls_uses.size());
+
+        for (int i = 0; i <= HCLS_ATTRS_STAT_MAX; i++) {
+            fprintf(stderr, "With % 3d attributes: %d\n", i, stats.num_hcls_by_attrs[i]);
+        }
+        fprintf(stderr, "With >% 2d attributes: %d\n", HCLS_ATTRS_STAT_MAX, stats.num_hcls_by_attrs_exceed);
+    }
+
+    fprintf(stderr, "\n");
 }
 
-void dumpHeapStatistics() {
-    global_heap.dumpHeapStatistics();
+void dumpHeapStatistics(int level) {
+    global_heap.dumpHeapStatistics(level);
 }
 
 //////

src/gc/heap.h

@@ -22,6 +22,7 @@
 
 #include "core/common.h"
 #include "core/threading.h"
+#include "core/types.h"
 
 namespace pyston {
 
@@ -535,13 +536,13 @@ public:
         huge_arena.freeUnmarked(weakly_referenced);
     }
 
-    void dumpHeapStatistics();
+    void dumpHeapStatistics(int level);
 
     friend void markPhase();
 };
 
 extern Heap global_heap;
-void dumpHeapStatistics();
+void dumpHeapStatistics(int level);
 
 } // namespace gc
 } // namespace pyston

src/runtime/stacktrace.cpp

@@ -160,28 +160,31 @@ extern "C" void abort() {
     // In case displaying the traceback recursively calls abort:
     static bool recursive = false;
 
-    // If traceback_cls is NULL, then we somehow died early on, and won't be able to display a traceback.
-    if (!recursive && traceback_cls) {
+    if (!recursive) {
         recursive = true;
-
+        Stats::dump();
         fprintf(stderr, "Someone called abort!\n");
 
-        // If we call abort(), things may be seriously wrong.  Set an alarm() to
-        // try to handle cases that we would just hang.
-        // (Ex if we abort() from a static constructor, and _printStackTrace uses
-        // that object, _printStackTrace will hang waiting for the first construction
-        // to finish.)
-        alarm(1);
-        try {
-            _printStacktrace();
-        } catch (ExcInfo) {
-            fprintf(stderr, "error printing stack trace during abort()");
-        }
+        // If traceback_cls is NULL, then we somehow died early on, and won't be able to display a traceback.
+        if (traceback_cls) {
+
+            // If we call abort(), things may be seriously wrong.  Set an alarm() to
+            // try to handle cases that we would just hang.
+            // (Ex if we abort() from a static constructor, and _printStackTrace uses
+            // that object, _printStackTrace will hang waiting for the first construction
+            // to finish.)
+            alarm(1);
+            try {
+                _printStacktrace();
+            } catch (ExcInfo) {
+                fprintf(stderr, "error printing stack trace during abort()");
+            }
 
-        // Cancel the alarm.
-        // This is helpful for when running in a debugger, since otherwise the debugger will catch the
-        // abort and let you investigate, but the alarm will still come back to kill the program.
-        alarm(0);
+            // Cancel the alarm.
+            // This is helpful for when running in a debugger, since otherwise the debugger will catch the
+            // abort and let you investigate, but the alarm will still come back to kill the program.
+            alarm(0);
+        }
     }
 
     if (PAUSE_AT_ABORT) {

test/tests/package_test.py

 import os
 
 import test_package
-print 1, test_package.__name__, os.path.normpath(test_package.__file__)
+print 1, test_package.__name__, os.path.normpath(test_package.__file__).replace(".pyc", ".py")
 
 import test_package.intrapackage_import
 import test_package.absolute_import

test/tests/test_package/absolute_import.py

 import os
 import import_target
 
-print 3, import_target.__name__, os.path.normpath(import_target.__file__)
+print 3, import_target.__name__, os.path.normpath(import_target.__file__).replace(".pyc", ".py")

test/tests/test_package/absolute_import_with_future.py

@@ -2,4 +2,4 @@ from __future__ import absolute_import
 import os
 import import_target
 
-print 4, import_target.__name__, os.path.normpath(import_target.__file__)
+print 4, import_target.__name__, os.path.normpath(import_target.__file__).replace(".pyc", ".py")

test/tests/test_package/intrapackage_import.py

 import os
 from . import import_target
-print 2, import_target.__name__, os.path.normpath(import_target.__file__)
+print 2, import_target.__name__, os.path.normpath(import_target.__file__).replace(".pyc", ".py")

test/unittests/analysis.cpp

@@ -39,7 +39,7 @@ TEST_F(AnalysisTest, augassign) {
     SourceInfo* si = new SourceInfo(createModule("augassign", fn), scoping, func, func->body, fn);
 
     CFG* cfg = computeCFG(si, func->body);
-    LivenessAnalysis* liveness = computeLivenessInfo(cfg);
+    std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
 
     //cfg->print();
 
@@ -49,7 +49,7 @@ TEST_F(AnalysisTest, augassign) {
             ASSERT_TRUE(liveness->isLiveAtEnd(module->interned_strings->get("a"), block));
     }
 
-    PhiAnalysis* phis = computeRequiredPhis(ParamNames(func), cfg, liveness, scope_info);
+    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(func), cfg, liveness.get(), scope_info);
 }
 
 void doOsrTest(bool is_osr, bool i_maybe_undefined) {
@@ -67,7 +67,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
             scoping, func, func->body, fn);
 
     CFG* cfg = computeCFG(si, func->body);
-    LivenessAnalysis* liveness = computeLivenessInfo(cfg);
+    std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
 
     // cfg->print();
 
@@ -83,7 +83,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     AST_Jump* backedge = ast_cast<AST_Jump>(loop_backedge->body[0]);
     ASSERT_LE(backedge->target->idx, loop_backedge->idx);
 
-    PhiAnalysis* phis;
+    std::unique_ptr<PhiAnalysis> phis;
 
     if (is_osr) {
         OSREntryDescriptor* entry_descriptor = OSREntryDescriptor::create(NULL, backedge);
@@ -91,9 +91,9 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
         if (i_maybe_undefined)
             entry_descriptor->args[idi_str] = NULL;
         entry_descriptor->args[iter_str] = NULL;
-        phis = computeRequiredPhis(entry_descriptor, liveness, scope_info);
+        phis = computeRequiredPhis(entry_descriptor, liveness.get(), scope_info);
     } else {
-        phis = computeRequiredPhis(ParamNames(func), cfg, liveness, scope_info);
+        phis = computeRequiredPhis(ParamNames(func), cfg, liveness.get(), scope_info);
     }
 
     // First, verify that we require phi nodes for the block we enter into.

tools/tester.py

@@ -222,8 +222,6 @@ def get_test_options(fn, check_stats, run_memcheck):
     return opts
 
 def determine_test_result(fn, opts, code, out, stderr, elapsed):
-    last_stderr_line = stderr.strip().split('\n')[-1]
-
     if opts.allow_warnings:
         out_lines = []
         for l in out.split('\n'):
@@ -235,13 +233,26 @@ def determine_test_result(fn, opts, code, out, stderr, elapsed):
         out = "\n".join(out_lines)
 
     stats = None
-    if code >= 0 and opts.collect_stats:
+    if opts.collect_stats:
         stats = {}
-        assert out.count("Stats:") == 1
-        out, stats_str = out.split("Stats:")
-        for l in stats_str.strip().split('\n'):
-            k, v = l.split(':')
-            stats[k.strip()] = int(v)
+        have_stats = (stderr.count("Stats:") == 1 and stderr.count("(End of stats)") == 1)
+
+        if code >= 0:
+            assert have_stats
+
+        if have_stats:
+            assert stderr.count("Stats:") == 1
+            stderr, stats_str = stderr.split("Stats:")
+            stats_str, stderr_tail = stats_str.split("(End of stats)\n")
+            stderr += stderr_tail
+
+            other_stats_str, counter_str = stats_str.split("Counters:")
+            for l in counter_str.strip().split('\n'):
+                assert l.count(':') == 1, l
+                k, v = l.split(':')
+                stats[k.strip()] = int(v)
+
+    last_stderr_line = stderr.strip().split('\n')[-1]
 
     if EXIT_CODE_ONLY:
         # fools the rest of this function into thinking the output is OK & just checking the exit code.