Merge pull request #814 from kmod/throw_capis3

Be able to jit functions that throw CAPI exceptions

minibenchmarks/django_template.py

 import os
 import sys
-sys.path.append(os.path.join(os.path.dirname(__file__), "../test/integration/django"))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../test/testsuite/lib/django"))
 
 from django.template.base import Origin, Template, Context, TemplateDoesNotExist
 from django.conf import settings

minibenchmarks/pyxl_bench.py

 import os
 import sys
 
-sys.path.append(os.path.join(os.path.dirname(__file__), "../test/integration/pyxl/"))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../test/testsuite/lib/pyxl/"))
 
 from pyxl.codec.register import pyxl_transform_string
 

minibenchmarks/sqlalchemy_declarative.py

@@ -2,7 +2,7 @@
 
 import os
 import sys
-sys.path.append(os.path.join(os.path.dirname(__file__), "../test/integration/sqlalchemy/lib"))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../test/testsuite/lib/sqlalchemy/lib"))
 
 from sqlalchemy import Column, ForeignKey, Integer, String
 from sqlalchemy.ext.declarative import declarative_base

minibenchmarks/sqlalchemy_imperative.py

 import os
 import sys
-sys.path.append(os.path.join(os.path.dirname(__file__), "../test/integration/sqlalchemy/lib"))
+sys.path.append(os.path.join(os.path.dirname(__file__), "../test/testsuite/lib/sqlalchemy/lib"))
 
 from sqlalchemy import Column, ForeignKey, Integer, String, Table, MetaData
 from sqlalchemy.ext.declarative import declarative_base

src/codegen/ast_interpreter.cpp

@@ -743,12 +743,19 @@ Box* ASTInterpreter::doOSR(AST_Jump* node) {
 
     UNAVOIDABLE_STAT_TIMER(t0, "us_timer_in_jitted_code");
     CompiledFunction* partial_func = compilePartialFuncInternal(&exit);
+
     auto arg_tuple = getTupleFromArgsArray(&arg_array[0], arg_array.size());
     Box* r = partial_func->call(std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
                                 std::get<3>(arg_tuple));
 
-    assert(r);
-    return r;
+    if (partial_func->exception_style == CXX) {
+        assert(r);
+        return r;
+    } else {
+        if (!r)
+            throwCAPIException();
+        return r;
+    }
 }
 
 Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
@@ -1725,14 +1732,24 @@ Box* astInterpretFunction(CLFunction* clfunc, int nargs, Box* closure, Box* gene
         clfunc->dependent_interp_callsites.invalidateAll();
 
         UNAVOIDABLE_STAT_TIMER(t0, "us_timer_in_jitted_code");
+        Box* r;
         if (closure && generator)
-            return optimized->closure_generator_call((BoxedClosure*)closure, (BoxedGenerator*)generator, arg1, arg2,
-                                                     arg3, args);
+            r = optimized->closure_generator_call((BoxedClosure*)closure, (BoxedGenerator*)generator, arg1, arg2, arg3,
+                                                  args);
         else if (closure)
-            return optimized->closure_call((BoxedClosure*)closure, arg1, arg2, arg3, args);
+            r = optimized->closure_call((BoxedClosure*)closure, arg1, arg2, arg3, args);
         else if (generator)
-            return optimized->generator_call((BoxedGenerator*)generator, arg1, arg2, arg3, args);
-        return optimized->call(arg1, arg2, arg3, args);
+            r = optimized->generator_call((BoxedGenerator*)generator, arg1, arg2, arg3, args);
+        else
+            r = optimized->call(arg1, arg2, arg3, args);
+
+        if (optimized->exception_style == CXX)
+            return r;
+        else {
+            if (!r)
+                throwCAPIException();
+            return r;
+        }
     }
 
     Box** vregs = NULL;

src/codegen/compvars.cpp

@@ -337,9 +337,7 @@ public:
                               CompilerVariable* slice) override {
         ConcreteCompilerVariable* converted_slice = slice->makeConverted(emitter, slice->getBoxType());
 
-        ExceptionStyle target_exception_style = CXX;
-        if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
-            target_exception_style = CAPI;
+        ExceptionStyle target_exception_style = info.preferredExceptionStyle();
 
         bool do_patchpoint = ENABLE_ICGETITEMS;
         llvm::Value* rtn;
@@ -500,9 +498,7 @@ CompilerVariable* UnknownType::getattr(IREmitter& emitter, const OpInfo& info, C
 
     llvm::Value* rtn_val = NULL;
 
-    ExceptionStyle target_exception_style = CXX;
-    if (info.unw_info.capi_exc_dest || (!cls_only && FORCE_LLVM_CAPI))
-        target_exception_style = CAPI;
+    ExceptionStyle target_exception_style = cls_only ? CXX : info.preferredExceptionStyle();
 
     llvm::Value* llvm_func;
     void* raw_func;
@@ -656,9 +652,7 @@ CompilerVariable* UnknownType::call(IREmitter& emitter, const OpInfo& info, Conc
     bool pass_keywords = (argspec.num_keywords != 0);
     int npassed_args = argspec.totalPassed();
 
-    ExceptionStyle exception_style = ((FORCE_LLVM_CAPI && !info.unw_info.cxx_exc_dest) || info.unw_info.capi_exc_dest)
-                                         ? ExceptionStyle::CAPI
-                                         : ExceptionStyle::CXX;
+    ExceptionStyle exception_style = info.preferredExceptionStyle();
 
     llvm::Value* func;
     if (pass_keywords)
@@ -691,10 +685,7 @@ CompilerVariable* UnknownType::callattr(IREmitter& emitter, const OpInfo& info,
     bool pass_keywords = (flags.argspec.num_keywords != 0);
     int npassed_args = flags.argspec.totalPassed();
 
-    ExceptionStyle exception_style = ((FORCE_LLVM_CAPI && !info.unw_info.cxx_exc_dest && !flags.null_on_nonexistent)
-                                      || info.unw_info.capi_exc_dest)
-                                         ? ExceptionStyle::CAPI
-                                         : ExceptionStyle::CXX;
+    ExceptionStyle exception_style = flags.null_on_nonexistent ? CXX : info.preferredExceptionStyle();
 
     if (exception_style == CAPI)
         assert(!flags.null_on_nonexistent); // Will conflict with CAPI's null-on-exception
@@ -1522,7 +1513,7 @@ public:
         if (canStaticallyResolveGetattrs()) {
             Box* rtattr = typeLookup(cls, attr, nullptr);
             if (rtattr == NULL) {
-                ExceptionStyle exception_style = (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest) ? CAPI : CXX;
+                ExceptionStyle exception_style = info.preferredExceptionStyle();
                 llvm::Value* raise_func = exception_style == CXX ? g.funcs.raiseAttributeErrorStr
                                                                  : g.funcs.raiseAttributeErrorStrCapi;
                 llvm::CallSite call = emitter.createCall3(
@@ -1619,15 +1610,13 @@ public:
                        "%d", info.unw_info.current_stmt->lineno);
 
         CompiledFunction* cf = NULL;
+        CompiledFunction* best_exception_mismatch = NULL;
         bool found = false;
         // TODO have to find the right version.. similar to resolveclfunc?
         for (int i = 0; i < cl->versions.size(); i++) {
             cf = cl->versions[i];
             assert(cf->spec->arg_types.size() == cl->numReceivedArgs());
 
-            if (cf->exception_style != exception_style)
-                continue;
-
             bool fits = true;
             for (int j = 0; j < args.size(); j++) {
                 if (!args[j]->canConvertTo(cf->spec->arg_types[j + 1])) {
@@ -1638,14 +1627,22 @@ public:
             if (!fits)
                 continue;
 
+            if (cf->exception_style != exception_style) {
+                if (!best_exception_mismatch)
+                    best_exception_mismatch = cf;
+                continue;
+            }
+
             found = true;
             break;
         }
 
-        if (!found && exception_style == CAPI) {
-            std::string name = g.func_addr_registry.getFuncNameAtAddress(cl->versions[0]->code, true);
-            RELEASE_ASSERT(0, "Please define a capi variant for %s", name.c_str());
+        if (!found) {
+            assert(best_exception_mismatch);
+            cf = best_exception_mismatch;
+            found = true;
         }
+
         RELEASE_ASSERT(found, "");
         RELEASE_ASSERT(cf->code, "");
 
@@ -1717,10 +1714,7 @@ public:
     CompilerVariable* callattr(IREmitter& emitter, const OpInfo& info, ConcreteCompilerVariable* var, BoxedString* attr,
                                CallattrFlags flags, const std::vector<CompilerVariable*>& args,
                                const std::vector<BoxedString*>* keyword_names) override {
-        ExceptionStyle exception_style = CXX;
-        // Not safe to force-capi here since most of the functions won't have capi variants:
-        if (/*FORCE_LLVM_CAPI ||*/ info.unw_info.capi_exc_dest)
-            exception_style = CAPI;
+        ExceptionStyle exception_style = info.preferredExceptionStyle();
 
         ConcreteCompilerVariable* called_constant = tryCallattrConstant(
             emitter, info, var, attr, flags.cls_only, flags.argspec, args, keyword_names, NULL, exception_style);
@@ -1783,9 +1777,7 @@ public:
         static BoxedString* attr = internStringImmortal("__getitem__");
         bool no_attribute = false;
 
-        ExceptionStyle exception_style = CXX;
-        if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
-            exception_style = CAPI;
+        ExceptionStyle exception_style = info.preferredExceptionStyle();
 
         ConcreteCompilerVariable* called_constant = tryCallattrConstant(
             emitter, info, var, attr, true, ArgPassSpec(1, 0, 0, 0), { slice }, NULL, &no_attribute, exception_style);
@@ -2293,9 +2285,7 @@ public:
                     rtn->incvref();
                     return rtn;
                 } else {
-                    ExceptionStyle target_exception_style = CXX;
-                    if (FORCE_LLVM_CAPI || info.unw_info.capi_exc_dest)
-                        target_exception_style = CAPI;
+                    ExceptionStyle target_exception_style = info.preferredExceptionStyle();
 
                     if (target_exception_style == CAPI) {
                         llvm::CallSite call = emitter.createCall(info.unw_info, g.funcs.raiseIndexErrorStrCapi,

src/codegen/irgen.cpp

@@ -495,6 +495,7 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
     std::unordered_map<CFGBlock*, ConcreteSymbolTable*> phi_ending_symbol_tables;
     typedef std::unordered_map<InternedString, std::pair<ConcreteCompilerType*, llvm::PHINode*>> PHITable;
     std::unordered_map<CFGBlock*, PHITable*> created_phis;
+    std::unordered_map<CFGBlock*, llvm::SmallVector<IRGenerator::ExceptionState, 2>> incoming_exception_state;
 
     CFGBlock* initial_block = NULL;
     if (entry_descriptor) {
@@ -759,6 +760,11 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
             }
         }
 
+        auto exc_it = incoming_exception_state.find(block);
+        if (exc_it != incoming_exception_state.end()) {
+            generator->setIncomingExceptionState(exc_it->second);
+        }
+
         // Generate loop safepoints on backedges.
         for (CFGBlock* predecessor : block->predecessors) {
             if (predecessor->idx > block->idx) {
@@ -777,6 +783,15 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
         phi_ending_symbol_tables[block] = ending_st.phi_symbol_table;
         llvm_exit_blocks[block] = ending_st.ending_block;
 
+        if (ending_st.exception_state.size()) {
+            AST_stmt* last_stmt = block->body.back();
+            assert(last_stmt->type == AST_TYPE::Invoke);
+            CFGBlock* exc_block = ast_cast<AST_Invoke>(last_stmt)->exc_dest;
+            assert(!incoming_exception_state.count(exc_block));
+
+            incoming_exception_state.insert(std::make_pair(exc_block, ending_st.exception_state));
+        }
+
         if (into_hax.count(block))
             ASSERT(ending_st.symbol_table->size() == 0, "%d", block->idx);
     }
@@ -952,7 +967,7 @@ static std::string getUniqueFunctionName(std::string nameprefix, EffortLevel eff
 
 CompiledFunction* doCompile(CLFunction* clfunc, SourceInfo* source, ParamNames* param_names,
                             const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                            FunctionSpecialization* spec, std::string nameprefix) {
+                            ExceptionStyle exception_style, FunctionSpecialization* spec, std::string nameprefix) {
     Timer _t("in doCompile");
     Timer _t2;
     long irgen_us = 0;
@@ -1015,8 +1030,7 @@ CompiledFunction* doCompile(CLFunction* clfunc, SourceInfo* source, ParamNames*
         }
     }
 
-
-    CompiledFunction* cf = new CompiledFunction(NULL, spec, NULL, effort, ExceptionStyle::CXX, entry_descriptor);
+    CompiledFunction* cf = new CompiledFunction(NULL, spec, NULL, effort, exception_style, entry_descriptor);
 
     // Make sure that the instruction memory keeps the module object alive.
     // TODO: implement this for real

src/codegen/irgen.h

@@ -20,6 +20,7 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IRBuilder.h"
 
+#include "core/options.h"
 #include "core/types.h"
 
 namespace pyston {
@@ -33,18 +34,18 @@ struct UnwindInfo {
 public:
     AST_stmt* current_stmt;
 
-    llvm::BasicBlock* capi_exc_dest;
-    llvm::BasicBlock* cxx_exc_dest;
+    llvm::BasicBlock* exc_dest;
 
-    bool hasHandler() const { return cxx_exc_dest != NULL || capi_exc_dest != NULL; }
+    bool hasHandler() const { return exc_dest != NULL; }
 
-    UnwindInfo(AST_stmt* current_stmt, llvm::BasicBlock* capi_exc_dest, llvm::BasicBlock* cxx_exc_dest)
-        : current_stmt(current_stmt), capi_exc_dest(capi_exc_dest), cxx_exc_dest(cxx_exc_dest) {}
+    UnwindInfo(AST_stmt* current_stmt, llvm::BasicBlock* exc_dest) : current_stmt(current_stmt), exc_dest(exc_dest) {}
+
+    ExceptionStyle preferredExceptionStyle() const;
 
     // Risky!  This means that we can't unwind from this location, and should be used in the
     // rare case that there are language-specific reasons that the statement should not unwind
     // (ex: loading function arguments into the appropriate scopes).
-    static UnwindInfo cantUnwind() { return UnwindInfo(NULL, NULL, NULL); }
+    static UnwindInfo cantUnwind() { return UnwindInfo(NULL, NULL); }
 };
 
 // TODO get rid of this
@@ -112,7 +113,7 @@ bool isIsDefinedName(llvm::StringRef name);
 
 CompiledFunction* doCompile(CLFunction* clfunc, SourceInfo* source, ParamNames* param_names,
                             const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                            FunctionSpecialization* spec, std::string nameprefix);
+                            ExceptionStyle exception_style, FunctionSpecialization* spec, std::string nameprefix);
 
 // A common pattern is to branch based off whether a variable is defined but only if it is
 // potentially-undefined.  If it is potentially-undefined, we have to generate control-flow
@@ -146,6 +147,8 @@ public:
         : effort(effort), type_recorder(type_recorder), unw_info(unw_info) {}
 
     TypeRecorder* getTypeRecorder() const { return type_recorder; }
+
+    ExceptionStyle preferredExceptionStyle() const { return unw_info.preferredExceptionStyle(); }
 };
 }
 

src/codegen/irgen/hooks.cpp

@@ -197,6 +197,12 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
 
     ASSERT(f->versions.size() < 20, "%s %ld", name.c_str(), f->versions.size());
 
+    ExceptionStyle exception_style = CXX;
+    if (FORCE_LLVM_CAPI_THROWS)
+        exception_style = CAPI;
+    if (name == "next")
+        exception_style = CAPI;
+
     if (VERBOSITY("irgen") >= 1) {
         std::string s;
         llvm::raw_string_ostream ss(s);
@@ -223,7 +229,8 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
             ss << "\033[" << colors[(int)effort] << ";1mDoing OSR-entry partial compile of " << source->fn << ":"
                << name << ", starting with backedge to block " << entry_descriptor->backedge->target->idx;
         }
-        ss << " at effort level " << (int)effort << '\n';
+        ss << " at effort level " << (int)effort << " with exception style "
+           << (exception_style == CXX ? "C++" : "CAPI") << '\n';
 
         if (entry_descriptor && VERBOSITY("irgen") >= 2) {
             for (const auto& p : entry_descriptor->args) {
@@ -241,8 +248,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
     }
 
 
-
-    CompiledFunction* cf = doCompile(f, source, &f->param_names, entry_descriptor, effort, spec, name);
+    CompiledFunction* cf = doCompile(f, source, &f->param_names, entry_descriptor, effort, exception_style, spec, name);
     compileIR(cf, effort);
 
     f->addVersion(cf);
@@ -822,7 +828,9 @@ extern "C" CompiledFunction* reoptCompiledFuncInternal(CompiledFunction* cf) {
 
 
 extern "C" char* reoptCompiledFunc(CompiledFunction* cf) {
-    return (char*)reoptCompiledFuncInternal(cf)->code;
+    CompiledFunction* new_cf = reoptCompiledFuncInternal(cf);
+    assert(new_cf->exception_style == cf->exception_style);
+    return (char*)new_cf->code;
 }
 
 CLFunction* createRTFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs,

src/codegen/irgen/irgenerator.h

@@ -72,8 +72,6 @@ private:
     llvm::Value* frame_info_arg;
     int scratch_size;
 
-    llvm::DenseMap<CFGBlock*, ExceptionStyle> landingpad_styles;
-
 public:
     IRGenState(CLFunction* clfunc, CompiledFunction* cf, SourceInfo* source_info, std::unique_ptr<PhiAnalysis> phis,
                ParamNames* param_names, GCBuilder* gc, llvm::MDNode* func_dbg_info);
@@ -82,6 +80,8 @@ public:
     CompiledFunction* getCurFunction() { return cf; }
     CLFunction* getCL() { return clfunc; }
 
+    ExceptionStyle getExceptionStyle() { return cf->exception_style; }
+
     llvm::Function* getLLVMFunction() { return cf->func; }
 
     EffortLevel getEffortLevel() { return cf->effort; }
@@ -107,15 +107,19 @@ public:
     ParamNames* getParamNames() { return param_names; }
 
     void setFrameInfoArgument(llvm::Value* v) { frame_info_arg = v; }
-
-    ExceptionStyle getLandingpadStyle(AST_Invoke* invoke);
-    ExceptionStyle getLandingpadStyle(CFGBlock* block);
 };
 
 // turns CFGBlocks into LLVM IR
 class IRGenerator {
 private:
 public:
+    struct ExceptionState {
+        llvm::BasicBlock* from_block;
+        ConcreteCompilerVariable* exc_type, *exc_value, *exc_tb;
+        ExceptionState(llvm::BasicBlock* from_block, ConcreteCompilerVariable* exc_type,
+                       ConcreteCompilerVariable* exc_value, ConcreteCompilerVariable* exc_tb)
+            : from_block(from_block), exc_type(exc_type), exc_value(exc_value), exc_tb(exc_tb) {}
+    };
     struct EndingState {
         // symbol_table records which Python variables are bound to what CompilerVariables at the end of this block.
         // phi_symbol_table records the ones that will need to be `phi'd.
@@ -123,8 +127,14 @@ public:
         SymbolTable* symbol_table;
         ConcreteSymbolTable* phi_symbol_table;
         llvm::BasicBlock* ending_block;
-        EndingState(SymbolTable* symbol_table, ConcreteSymbolTable* phi_symbol_table, llvm::BasicBlock* ending_block)
-            : symbol_table(symbol_table), phi_symbol_table(phi_symbol_table), ending_block(ending_block) {}
+        llvm::SmallVector<ExceptionState, 2> exception_state;
+
+        EndingState(SymbolTable* symbol_table, ConcreteSymbolTable* phi_symbol_table, llvm::BasicBlock* ending_block,
+                    llvm::ArrayRef<ExceptionState> exception_state)
+            : symbol_table(symbol_table),
+              phi_symbol_table(phi_symbol_table),
+              ending_block(ending_block),
+              exception_state(exception_state.begin(), exception_state.end()) {}
     };
 
     virtual ~IRGenerator() {}
@@ -139,6 +149,10 @@ public:
     virtual void doSafePoint(AST_stmt* next_statement) = 0;
     virtual void addFrameStackmapArgs(PatchpointInfo* pp, AST_stmt* current_stmt,
                                       std::vector<llvm::Value*>& stackmap_args) = 0;
+    virtual void addOutgoingExceptionState(ExceptionState exception_state) = 0;
+    virtual void setIncomingExceptionState(llvm::SmallVector<ExceptionState, 2> exc_state) = 0;
+    virtual llvm::BasicBlock* getCXXExcDest(llvm::BasicBlock* final_dest) = 0;
+    virtual llvm::BasicBlock* getCAPIExcDest(llvm::BasicBlock* final_dest, AST_stmt* current_stmt) = 0;
 };
 
 class IREmitter;

src/codegen/runtime_hooks.cpp

@@ -311,6 +311,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(raise3_capi);
     GET(PyErr_Fetch);
     GET(PyErr_NormalizeException);
+    GET(PyErr_Restore);
     GET(capiExcCaughtInJit);
     GET(reraiseJitCapiExc);
     GET(deopt);

src/codegen/runtime_hooks.h

@@ -51,7 +51,7 @@ struct GlobalFuncs {
 
     llvm::Value* __cxa_end_catch;
     llvm::Value* raise0, *raise3, *raise3_capi;
-    llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *capiExcCaughtInJit, *reraiseJitCapiExc;
+    llvm::Value* PyErr_Fetch, *PyErr_NormalizeException, *PyErr_Restore, *capiExcCaughtInJit, *reraiseJitCapiExc;
     llvm::Value* deopt;
 
     llvm::Value* div_float_float, *floordiv_float_float, *mod_float_float, *pow_float_float;

src/core/options.cpp

@@ -45,8 +45,10 @@ bool USE_REGALLOC_BASIC = true;
 bool PAUSE_AT_ABORT = false;
 bool ENABLE_TRACEBACKS = true;
 // Forces the llvm jit to use capi exceptions whenever it can, as opposed to whenever it thinks
-// it is faster:
-bool FORCE_LLVM_CAPI = false;
+// it is faster.  The CALLS version is for calls that the llvm jit will make, and the THROWS version
+// is for the exceptions it will throw.
+bool FORCE_LLVM_CAPI_CALLS = false;
+bool FORCE_LLVM_CAPI_THROWS = false;
 
 int OSR_THRESHOLD_INTERPRETER = 25;
 int REOPT_THRESHOLD_INTERPRETER = 25;

src/core/options.h

@@ -39,7 +39,7 @@ extern int MAX_OBJECT_CACHE_ENTRIES;
 
 extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
     CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
-    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI;
+    PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING, FORCE_LLVM_CAPI_CALLS, FORCE_LLVM_CAPI_THROWS;
 
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,

src/runtime/generator.cpp

@@ -117,19 +117,33 @@ Box* generatorIter(Box* s) {
 }
 
 // called from both generatorHasNext and generatorSend/generatorNext (but only if generatorHasNext hasn't been called)
-static void generatorSendInternal(BoxedGenerator* self, Box* v) {
+template <ExceptionStyle S> static bool generatorSendInternal(BoxedGenerator* self, Box* v) noexcept(S == CAPI) {
     STAT_TIMER(t0, "us_timer_generator_switching", 0);
 
-    if (!self->returnContext && v != None)
-        raiseExcHelper(TypeError, "can't send non-None value to a just-started generator");
+    if (!self->returnContext && v != None) {
+        if (S == CAPI) {
+            PyErr_SetString(TypeError, "can't send non-None value to a just-started generator");
+            return true;
+        } else
+            raiseExcHelper(TypeError, "can't send non-None value to a just-started generator");
+    }
 
-    if (self->running)
-        raiseExcHelper(ValueError, "generator already executing");
+    if (self->running) {
+        if (S == CAPI) {
+            PyErr_SetString(ValueError, "generator already executing");
+            return true;
+        } else
+            raiseExcHelper(ValueError, "generator already executing");
+    }
 
     // check if the generator already exited
     if (self->entryExited) {
         freeGeneratorStack(self);
-        raiseExcHelper(StopIteration, (const char*)nullptr);
+        if (S == CAPI) {
+            PyErr_SetObject(StopIteration, None);
+            return true;
+        } else
+            raiseExcHelper(StopIteration, (const char*)nullptr);
     }
 
     self->returnValue = v;
@@ -158,9 +172,14 @@ static void generatorSendInternal(BoxedGenerator* self, Box* v) {
     if (self->exception.type) {
         freeGeneratorStack(self);
         // don't raise StopIteration exceptions because those are handled specially.
-        if (!self->exception.matches(StopIteration))
-            throw self->exception;
-        return;
+        if (!self->exception.matches(StopIteration)) {
+            if (S == CAPI) {
+                setCAPIException(self->exception);
+                return true;
+            } else
+                throw self->exception;
+        }
+        return false;
     }
 
     if (self->entryExited) {
@@ -169,18 +188,21 @@ static void generatorSendInternal(BoxedGenerator* self, Box* v) {
         // We could directly create the StopIteration exception but we delay creating it because often the caller is not
         // interested in the exception (=generatorHasnext). If we really need it we will create it inside generatorSend.
         self->exception = ExcInfo(NULL, NULL, NULL);
-        return;
+        return false;
     }
+    return false;
 }
 
-Box* generatorSend(Box* s, Box* v) {
+template <ExceptionStyle S> static Box* generatorSend(Box* s, Box* v) noexcept(S == CAPI) {
     assert(s->cls == generator_cls);
     BoxedGenerator* self = static_cast<BoxedGenerator*>(s);
 
     if (self->iterated_from__hasnext__)
         Py_FatalError(".throw called on generator last advanced with __hasnext__");
 
-    generatorSendInternal(self, v);
+    bool exc = generatorSendInternal<S>(self, v);
+    if (S == CAPI && exc)
+        return NULL;
 
     // throw StopIteration if the generator exited
     if (self->entryExited) {
@@ -195,9 +217,19 @@ Box* generatorSend(Box* s, Box* v) {
         ExcInfo old_exc = self->exception;
         // Clear the exception for GC purposes:
         self->exception = ExcInfo(nullptr, nullptr, nullptr);
-        if (old_exc.type == NULL)
-            raiseExcHelper(StopIteration, (const char*)nullptr);
-        throw old_exc;
+        if (old_exc.type == NULL) {
+            if (S == CAPI) {
+                PyErr_SetObject(StopIteration, None);
+                return NULL;
+            } else
+                raiseExcHelper(StopIteration, (const char*)nullptr);
+        } else {
+            if (S == CAPI) {
+                setCAPIException(old_exc);
+                return NULL;
+            } else
+                throw old_exc;
+        }
     }
 
     return self->returnValue;
@@ -223,7 +255,7 @@ Box* generatorThrow(Box* s, BoxedClass* exc_cls, Box* exc_val = nullptr, Box** a
         throw exc_info;
 
     self->exception = exc_info;
-    return generatorSend(self, None);
+    return generatorSend<CXX>(self, None);
 }
 
 Box* generatorClose(Box* s) {
@@ -245,7 +277,7 @@ Box* generatorClose(Box* s) {
     assert(0); // unreachable
 }
 
-Box* generatorNext(Box* s) {
+template <ExceptionStyle S> static Box* generatorNext(Box* s) noexcept(S == CAPI) {
     assert(s->cls == generator_cls);
     BoxedGenerator* self = static_cast<BoxedGenerator*>(s);
 
@@ -254,7 +286,7 @@ Box* generatorNext(Box* s) {
         return self->returnValue;
     }
 
-    return generatorSend(s, None);
+    return generatorSend<S>(s, None);
 }
 
 i1 generatorHasnextUnboxed(Box* s) {
@@ -262,7 +294,7 @@ i1 generatorHasnextUnboxed(Box* s) {
     BoxedGenerator* self = static_cast<BoxedGenerator*>(s);
 
     if (!self->iterated_from__hasnext__) {
-        generatorSendInternal(self, None);
+        generatorSendInternal<CXX>(self, None);
         self->iterated_from__hasnext__ = true;
     }
 
@@ -455,13 +487,16 @@ void setupGenerator() {
                             new BoxedFunction(boxRTFunction((void*)generatorIter, typeFromClass(generator_cls), 1)));
 
     generator_cls->giveAttr("close", new BoxedFunction(boxRTFunction((void*)generatorClose, UNKNOWN, 1)));
-    generator_cls->giveAttr("next", new BoxedFunction(boxRTFunction((void*)generatorNext, UNKNOWN, 1)));
+
+    auto generator_next = boxRTFunction((void*)generatorNext<CXX>, UNKNOWN, 1, ParamNames::empty(), CXX);
+    addRTFunction(generator_next, (void*)generatorNext<CAPI>, UNKNOWN, CAPI);
+    generator_cls->giveAttr("next", new BoxedFunction(generator_next));
 
     CLFunction* hasnext = boxRTFunction((void*)generatorHasnextUnboxed, BOOL, 1);
     addRTFunction(hasnext, (void*)generatorHasnext, BOXED_BOOL);
     generator_cls->giveAttr("__hasnext__", new BoxedFunction(hasnext));
 
-    generator_cls->giveAttr("send", new BoxedFunction(boxRTFunction((void*)generatorSend, UNKNOWN, 2)));
+    generator_cls->giveAttr("send", new BoxedFunction(boxRTFunction((void*)generatorSend<CXX>, UNKNOWN, 2)));
     auto gthrow = new BoxedFunction(boxRTFunction((void*)generatorThrow, UNKNOWN, 4, 2, false, false), { NULL, NULL });
     generator_cls->giveAttr("throw", gthrow);
 

src/runtime/inline/link_forcer.cpp

@@ -129,6 +129,7 @@ void force() {
     FORCE(raise3_capi);
     FORCE(PyErr_Fetch);
     FORCE(PyErr_NormalizeException);
+    FORCE(PyErr_Restore);
     FORCE(capiExcCaughtInJit);
     FORCE(reraiseJitCapiExc);
     FORCE(deopt);

src/runtime/util.cpp

@@ -262,6 +262,10 @@ extern "C" void dumpEx(void* p, int levels) {
             printf("Has %ld function versions\n", cl->versions.size());
             for (CompiledFunction* cf : cl->versions) {
                 bool got_name;
+                if (cf->exception_style == CXX)
+                    printf("CXX style: ");
+                else
+                    printf("CAPI style: ");
                 std::string name = g.func_addr_registry.getFuncNameAtAddress(cf->code, true, &got_name);
                 if (got_name)
                     printf("%s\n", name.c_str());