Commit 6521ad76 authored by Boxiang Sun's avatar Boxiang Sun Committed by Xavier Thompson

Boxiang: Create nogil extension to cdef classes

parent d95e0e1d
......@@ -2050,6 +2050,8 @@ class CCodeWriter(object):
self.putln("%s = NULL;" % decl)
elif type.is_memoryviewslice:
self.putln("%s = %s;" % (decl, type.literal_code(type.default_value)))
elif type.is_struct and type.is_extension_type and type.nogil:
self.putln("%s;" % decl)
else:
self.putln("%s%s;" % (static and "static " or "", decl))
......
......@@ -704,6 +704,8 @@ class ExprNode(Node):
self.gil_error()
def gil_assignment_check(self, env):
if self.type == PyrexTypes.PyExtensionType and env.nogil and self.type.nogil:
error("No gil type in no gil function")
if env.nogil and self.type.is_pyobject:
error(self.pos, "Assignment of Python object not allowed without gil")
......@@ -1719,6 +1721,9 @@ class StringNode(PyConstNode):
is_identifier = None
unicode_value = None
# XXX: Let the StringNode can be used in nogil extension initializing
nogil_check = None
def calculate_constant_result(self):
if self.unicode_value is not None:
# only the Unicode value is portable across Py2/3
......@@ -2314,6 +2319,9 @@ class NameNode(AtomicExprNode):
code.error_goto_if_null(self.result(), self.pos)))
self.generate_gotref(code)
elif entry.is_local and isinstance(entry.type, PyrexTypes.CythonExtensionType):
pass
# code.putln(entry.cname)
elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice:
# Raise UnboundLocalError for objects and memoryviewslices
raise_unbound = (
......@@ -5547,6 +5555,18 @@ class CallNode(ExprNode):
self.analyse_c_function_call(env)
self.type = type
return True
elif type and type.is_struct and type.nogil:
args, kwds = self.explicit_args_kwds()
items = []
for arg, member in zip(args, type.scope.var_entries):
items.append(DictItemNode(pos=arg.pos, key=StringNode(pos=arg.pos, value=member.name), value=arg))
if kwds:
items += kwds.key_value_pairs
self.key_value_pairs = items
self.__class__ = DictNode
self.analyse_types(env) # FIXME
self.coerce_to(type, env)
return True
def is_lvalue(self):
return self.type.is_reference
......@@ -7203,9 +7223,12 @@ class AttributeNode(ExprNode):
# (AnalyseExpressionsTransform)
self.member = self.entry.cname
return "((struct %s *)%s%s%s)->%s" % (
obj.type.vtabstruct_cname, obj_code, self.op,
obj.type.vtabslot_cname, self.member)
if obj.type.nogil:
return "%s" % self.entry.func_cname
else:
return "((struct %s *)%s%s%s)->%s" % (
obj.type.vtabstruct_cname, obj_code, self.op,
obj.type.vtabslot_cname, self.member)
elif self.result_is_used:
return self.member
# Generating no code at all for unused access to optimised builtin
......@@ -8804,6 +8827,8 @@ class DictNode(ExprNode):
# pairs are evaluated and used one at a time.
code.mark_pos(self.pos)
self.allocate_temp_result(code)
if hasattr(self.type, 'nogil') and self.type.nogil:
code.putln("%s = (struct %s *)malloc(sizeof(struct %s));" % (self.result(), self.type.objstruct_cname, self.type.objstruct_cname))
is_dict = self.type.is_pyobject
if is_dict:
......@@ -8861,6 +8886,11 @@ class DictNode(ExprNode):
code.putln('}')
if self.exclude_null_values:
code.putln('}')
elif self.type.nogil:
code.putln("%s->%s = %s;" % (
self.result(),
item.key.value,
item.value.result()))
else:
code.putln("%s.%s = %s;" % (
self.result(),
......
......@@ -1197,6 +1197,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
self.sue_header_footer(type, "struct", type.objstruct_cname)
code.putln(header)
base_type = type.base_type
nogil = type.nogil
if base_type:
basestruct_cname = base_type.objstruct_cname
if basestruct_cname == "PyTypeObject":
......@@ -1207,6 +1208,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
("struct ", "")[base_type.typedef_flag],
basestruct_cname,
Naming.obj_base_cname))
elif nogil:
# Extension type with nogil keyword indicate it is a CPython-free struct
code.globalstate.use_utility_code(
UtilityCode.load_cached("CythonReferenceCounting", "ObjectHandling.c"))
code.putln(
"// nogil"
)
code.putln(
"int ob_refcnt;" # "CyObject_HEAD;" Sometimes the CythonReferenceCounting was put after the nogil extension declaration, WTF!!!
)
else:
code.putln(
"PyObject_HEAD")
......
......@@ -3376,6 +3376,8 @@ class DefNodeWrapper(FuncDefNode):
# different code types.
for arg in self.args:
if not arg.type.is_pyobject:
if arg.type is PyrexTypes.PyExtensionType and arg.type.nogil:
continue # XXX maybe here is not the correct place to put it...
if not arg.type.create_from_py_utility_code(env):
pass # will fail later
elif arg.hdr_type and not arg.hdr_type.is_pyobject:
......@@ -4906,6 +4908,7 @@ class CClassDefNode(ClassDefNode):
# doc string or None
# body StatNode or None
# entry Symtab.Entry
# nogil boolean
# base_type PyExtensionType or None
# buffer_defaults_node DictNode or None Declares defaults for a buffer
# buffer_defaults_pos
......@@ -4915,6 +4918,7 @@ class CClassDefNode(ClassDefNode):
buffer_defaults_pos = None
typedef_flag = False
api = False
nogil = False
objstruct_name = None
typeobj_name = None
check_size = None
......@@ -4959,6 +4963,7 @@ class CClassDefNode(ClassDefNode):
typedef_flag=self.typedef_flag,
check_size = self.check_size,
api=self.api,
nogil=self.nogil,
buffer_defaults=self.buffer_defaults(env),
shadow=self.shadow)
......@@ -5047,6 +5052,7 @@ class CClassDefNode(ClassDefNode):
visibility=self.visibility,
typedef_flag=self.typedef_flag,
api=self.api,
nogil=self.nogil,
buffer_defaults=self.buffer_defaults(env),
shadow=self.shadow)
......@@ -6186,6 +6192,8 @@ class DelStatNode(StatNode):
error(arg.pos, "Deletion of global C variable")
elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
self.cpp_check(env)
elif arg.type.is_struct_or_union and arg.type.nogil:
pass # del nogil extension
elif arg.type.is_cpp_class:
error(arg.pos, "Deletion of non-heap C++ object")
elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
......@@ -6214,6 +6222,8 @@ class DelStatNode(StatNode):
arg.generate_evaluation_code(code)
code.putln("delete %s;" % arg.result())
arg.generate_disposal_code(code)
elif arg.type.is_struct_or_union and hasattr(arg.type, "nogil") and arg.type.nogil:
code.putln("free(&%s);" % arg.result())
# else error reported earlier
def annotate(self, code):
......
......@@ -1720,6 +1720,7 @@ if VALUE is not None:
if stats:
node.body.stats += stats
if (node.visibility != 'extern'
and not node.nogil
and not node.scope.lookup('__reduce__')
and not node.scope.lookup('__reduce_ex__')):
self._inject_pickle_methods(node)
......
......@@ -2235,7 +2235,7 @@ def p_statement(s, ctx, first_statement = 0):
elif s.sy == 'IF':
return p_IF_statement(s, ctx)
elif s.sy == '@':
if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'):
if ctx.level not in ('module', 'class', 'c_class', 'class_nogil', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'):
s.error('decorator not allowed here')
s.level = ctx.level
decorators = p_decorators(s)
......@@ -2253,11 +2253,14 @@ def p_statement(s, ctx, first_statement = 0):
cdef_flag = 1
s.next()
elif s.sy == 'cpdef':
if ctx.level == 'c_class_nogil':
s.error('cpdef statement not allowed in nogil extension type')
s.level = ctx.level
cdef_flag = 1
overridable = 1
s.next()
if cdef_flag:
if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_nogil', 'c_class_pxd'):
s.error('cdef statement not allowed here')
s.level = ctx.level
node = p_cdef_statement(s, ctx(overridable=overridable))
......@@ -2277,6 +2280,8 @@ def p_statement(s, ctx, first_statement = 0):
# as part of a cdef class
if ('pxd' in ctx.level) and (ctx.level != 'c_class_pxd'):
s.error('def statement not allowed here')
if ctx.level == 'c_class_nogil':
s.error('def statement not allowed in nogil extension type, only cdef with nogil is allowed')
s.level = ctx.level
return p_def_statement(s, decorators)
elif s.sy == 'class':
......@@ -2287,7 +2292,7 @@ def p_statement(s, ctx, first_statement = 0):
if ctx.level not in ('module', 'module_pxd'):
s.error("include statement not allowed here")
return p_include_statement(s, ctx)
elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
elif ctx.level in ('c_class', 'c_class_nogil') and s.sy == 'IDENT' and s.systring == 'property':
return p_property_decl(s)
elif s.sy == 'pass' and ctx.level != 'property':
return p_pass_statement(s, with_newline=True)
......@@ -3271,7 +3276,7 @@ def p_c_modifiers(s):
return []
def p_c_func_or_var_declaration(s, pos, ctx):
cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
cmethod_flag = ctx.level in ('c_class', 'c_class_pxd', 'c_class_nogil')
modifiers = p_c_modifiers(s)
base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates)
declarator = p_c_declarator(s, ctx(modifiers=modifiers), cmethod_flag = cmethod_flag,
......@@ -3290,8 +3295,13 @@ def p_c_func_or_var_declaration(s, pos, ctx):
fatal=False)
s.next()
p_test(s) # Keep going, but ignore result.
if s.sy == 'nogil':
nogil = p_nogil(s)
s.next()
if ctx.level == 'c_class_nogil' and not nogil:
s.error("Only C function with nogil allowed in nogil extension")
if s.sy == ':':
if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates:
if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class', 'c_class_nogil') and not ctx.templates:
s.error("C function definition not allowed here")
doc, suite = p_suite_with_docstring(s, Ctx(level='function'))
result = Nodes.CFuncDefNode(pos,
......@@ -3319,7 +3329,7 @@ def p_c_func_or_var_declaration(s, pos, ctx):
declarators.append(declarator)
doc_line = s.start_line + 1
s.expect_newline("Syntax error in C variable declaration", ignore_semicolon=True)
if ctx.level in ('c_class', 'c_class_pxd') and s.start_line == doc_line:
if ctx.level in ('c_class', 'c_class_pxd', 'c_class_nogil') and s.start_line == doc_line:
doc = p_doc_string(s)
else:
doc = None
......@@ -3520,9 +3530,12 @@ def p_c_class_definition(s, pos, ctx):
if ctx.visibility not in ('public', 'extern') and not ctx.api:
error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class")
objstruct_name, typeobj_name, check_size = p_c_class_options(s)
nogil = p_nogil(s)
if s.sy == ':':
if ctx.level == 'module_pxd':
body_level = 'c_class_pxd'
elif nogil:
body_level = 'c_class_nogil'
else:
body_level = 'c_class'
doc, body = p_suite_with_docstring(s, Ctx(level=body_level))
......@@ -3561,7 +3574,8 @@ def p_c_class_definition(s, pos, ctx):
check_size = check_size,
in_pxd = ctx.level == 'module_pxd',
doc = doc,
body = body)
body = body,
nogil = nogil or ctx.nogil)
def p_c_class_options(s):
......
......@@ -1325,6 +1325,215 @@ class PyObjectType(PyrexType):
return cname
class CythonObjectType(PyrexType):
#
# Base class for all nogil extension object types (reference-counted).
#
name = "cythonobject"
is_pyobject = 0
default_value = "0"
declaration_value = "0"
buffer_defaults = None
is_extern = False
is_subclassed = False
is_gc_simple = False
def __str__(self):
return "Cython object"
def __repr__(self):
return "<CythonObjectType>"
def can_coerce_to_pyobject(self, env):
return False
def can_coerce_from_pyobject(self, env):
return False
def default_coerced_ctype(self):
"""The default C type that this Python type coerces to, or None."""
return None
def assignable_from(self, src_type):
# except for pointers, conversion will be attempted
# return not src_type.is_ptr or src_type.is_string or src_type.is_pyunicode_ptr
return False
def declaration_code(self, entity_code,
for_display = 0, dll_linkage = None, pyrex = 0):
raise NotImplementedError("Calling declartion_code on Cython object")
if pyrex or for_display:
base_code = "object"
else:
base_code = public_decl("PyObject", dll_linkage)
entity_code = "*%s" % entity_code
return self.base_declaration_code(base_code, entity_code)
def as_pyobject(self, cname):
raise NotImplementedError("Calling as_pyobject on Cython object")
if (not self.is_complete()) or self.is_extension_type:
return "(PyObject *)" + cname
else:
return cname
def py_type_name(self):
raise NotImplementedError("Calling py_type_name on Cython object")
return "cythonobject"
def __lt__(self, other):
"""
Make sure we sort highest, as instance checking on py_type_name
('object') is always true
"""
return False
def global_init_code(self, entry, code):
raise NotImplementedError("Calling global_init_code on Cython object")
code.put_init_var_to_py_none(entry, nanny=False)
def check_for_null_code(self, cname):
raise NotImplementedError("Calling check_for_null_code on Cython object")
return cname
class CythonExtensionType(CythonObjectType):
#
# A Cython extension type with nogil flag
# TODO: This type may need big amend
#
# name string
# scope CClassScope Attribute namespace
# visibility string
# typedef_flag boolean
# base_type PyExtensionType or None
# nogil boolean
# module_name string or None Qualified name of defining module
# objstruct_cname string Name of PyObject struct
# objtypedef_cname string Name of PyObject struct typedef
# typeobj_cname string or None C code fragment referring to type object
# typeptr_cname string or None Name of pointer to external type object
# vtabslot_cname string Name of C method table member
# vtabstruct_cname string Name of C method table struct
# vtabptr_cname string Name of pointer to C method table
# vtable_cname string Name of C method table definition
# early_init boolean Whether to initialize early (as opposed to during module execution).
# defered_declarations [thunk] Used to declare class hierarchies in order
# check_size 'warn', 'error', 'ignore' What to do if tp_basicsize does not match
is_extension_type = 1
is_struct = 1
is_struct_or_union = 1
nogil = 1
is_pyobject = 0
has_attributes = 1
early_init = 1
objtypedef_cname = None
def __init__(self, name, typedef_flag, base_type, is_external=0, check_size=None):
self.name = name
self.scope = None
self.typedef_flag = typedef_flag
if base_type is not None:
base_type.is_subclassed = True
self.base_type = base_type
self.nogil = False
self.module_name = None
self.objstruct_cname = None
self.typeobj_cname = None
self.typeptr_cname = None
self.vtabslot_cname = None
self.vtabstruct_cname = None
self.vtabptr_cname = None
self.vtable_cname = None
self.is_external = is_external
self.check_size = check_size or 'warn'
self.defered_declarations = []
def set_scope(self, scope):
self.scope = scope
if scope:
scope.parent_type = self
def needs_nonecheck(self):
return True
def subtype_of_resolved_type(self, other_type):
if other_type.is_extension_type or other_type.is_builtin_type:
return self is other_type or (
self.base_type and self.base_type.subtype_of(other_type))
else:
return other_type is py_object_type
def typeobj_is_available(self):
# Do we have a pointer to the type object?
return self.typeptr_cname
def typeobj_is_imported(self):
# If we don't know the C name of the type object but we do
# know which module it's defined in, it will be imported.
return self.typeobj_cname is None and self.module_name is not None
def assignable_from(self, src_type):
if self == src_type:
return True
if isinstance(src_type, PyExtensionType):
if src_type.base_type is not None:
return self.assignable_from(src_type.base_type)
if isinstance(src_type, BuiltinObjectType):
# FIXME: This is an ugly special case that we currently
# keep supporting. It allows users to specify builtin
# types as external extension types, while keeping them
# compatible with the real builtin types. We already
# generate a warning for it. Big TODO: remove!
return (self.module_name == '__builtin__' and
self.name == src_type.name)
return False
def declaration_code(self, entity_code,
for_display = 0, dll_linkage = None, pyrex = 0, deref = 0):
if pyrex or for_display:
base_code = self.name
else:
if self.typedef_flag:
objstruct = self.objstruct_cname
else:
objstruct = "struct %s" % self.objstruct_cname
base_code = public_decl(objstruct, dll_linkage)
if deref:
assert not entity_code
else:
entity_code = "*%s" % entity_code
return self.base_declaration_code(base_code, entity_code)
def type_test_code(self, py_arg, notnone=False):
none_check = "((%s) == Py_None)" % py_arg
type_check = "likely(__Pyx_TypeTest(%s, %s))" % (
py_arg, self.typeptr_cname)
if notnone:
return type_check
else:
return "likely(%s || %s)" % (none_check, type_check)
def attributes_known(self):
return self.scope is not None
def __str__(self):
return self.name
def __repr__(self):
return "<CythonExtensionType %s%s>" % (self.scope.class_name,
("", " typedef")[self.typedef_flag])
def py_type_name(self):
if not self.module_name:
return self.name
return "__import__(%r, None, None, ['']).%s" % (self.module_name,
self.name)
builtin_types_that_cannot_create_refcycles = set([
'object', 'bool', 'int', 'long', 'float', 'complex',
'bytearray', 'bytes', 'unicode', 'str', 'basestring'
......@@ -1483,6 +1692,7 @@ class PyExtensionType(PyObjectType):
# visibility string
# typedef_flag boolean
# base_type PyExtensionType or None
# nogil boolean
# module_name string or None Qualified name of defining module
# objstruct_cname string Name of PyObject struct
# objtypedef_cname string Name of PyObject struct typedef
......@@ -1509,6 +1719,7 @@ class PyExtensionType(PyObjectType):
if base_type is not None:
base_type.is_subclassed = True
self.base_type = base_type
self.nogil = False
self.module_name = None
self.objstruct_cname = None
self.typeobj_cname = None
......
......@@ -1573,7 +1573,7 @@ class ModuleScope(Scope):
def declare_c_class(self, name, pos, defining=0, implementing=0,
module_name=None, base_type=None, objstruct_cname=None,
typeobj_cname=None, typeptr_cname=None, visibility='private',
typedef_flag=0, api=0, check_size=None,
typedef_flag=0, api=0, nogil=0, check_size=None,
buffer_defaults=None, shadow=0):
# If this is a non-extern typedef class, expose the typedef, but use
# the non-typedef struct internally to avoid needing forward
......@@ -1606,8 +1606,15 @@ class ModuleScope(Scope):
# Make a new entry if needed
#
if not entry or shadow:
type = PyrexTypes.PyExtensionType(
name, typedef_flag, base_type, visibility == 'extern', check_size=check_size)
if nogil:
pass
if nogil:
type = PyrexTypes.CythonExtensionType(
name, typedef_flag, base_type, visibility == 'extern', check_size=check_size)
else:
type = PyrexTypes.PyExtensionType(
name, typedef_flag, base_type, visibility == 'extern', check_size=check_size)
type.nogil = nogil
type.pos = pos
type.buffer_defaults = buffer_defaults
if objtypedef_cname is not None:
......@@ -2327,7 +2334,8 @@ class CClassScope(ClassScope):
defining=0, modifiers=(), utility_code=None, overridable=False):
name = self.mangle_class_private_name(name)
if get_special_method_signature(name) and not self.parent_type.is_builtin_type:
error(pos, "Special methods must be declared with 'def', not 'cdef'")
if not(hasattr(self.parent_type, "nogil") and self.parent_type.nogil and self.parent_type.is_struct_or_union):
error(pos, "Special methods must be declared with 'def', not 'cdef'")
args = type.args
if not type.is_static_method:
if not args:
......
......@@ -1851,6 +1851,51 @@ try_unpack:
return 0;
}
/////////////// CythonReferenceCounting.proto ///////////////
#include <stdlib.h>
#include <stddef.h>
#if !defined(__GNUC__)
#define GCC_VERSION (__GNUC__ * 10000 \
+ __GNUC_MINOR__ * 100 \
+ __GNUC_PATCHLEVEL__)
/* Test for GCC > 4.9.0 */
#if GCC_VERSION < 40900
#error atomic.h works only with GCC newer than version 4.9
#endif /* GNUC >= 4.9 */
#endif /* Has GCC */
// #include <stdatomic.h>
/* CyObject_HEAD defines the initial segment of every CyObject. */
#define CyObject_HEAD \
int ob_refcnt; \
void (*cdealloc)();
struct CyObject {
CyObject_HEAD
};
/* Cast argument to PyObject* type. */
#define _CyObject_CAST(op) ((struct CyObject*)(op))
// XXX: Without scope analysis this is useless...
/*
static inline void _Cy_DECREF(struct CyObject *op) {
if (atomic_fetch_sub(&(op->ob_refcnt), 1) == 1) {
op->cdealloc(op);
}
}
static inline void _Cy_INCREF(struct CyObject *op) {
atomic_fetch_add(&(op->ob_refcnt), 1);
}
#define Cy_INCREF(op) _Cy_INCREF(_CyObject_CAST(op))
#define Cy_DECREF(op) _Cy_DECREF(_CyObject_CAST(op))
*/
/////////////// UnpackUnboundCMethod.proto ///////////////
......
import subprocess
import importlib
import build_extension
def run(env, python_path):
source = subprocess.check_output([python_path + " -c 'import nogil_extension; a = nogil_extension.bag(); print(a)'"],
env=env,
shell=True,
)
failure_count = 0
if source != b'4.0\n42.0\n':
failure_count = 1
result_dict = {'failed': failure_count, 'stdout': source}
return result_dict
import subprocess
import importlib
import sys
def build_ext(env):
# Build the cython_lwan_coro when installing it in slapos
# subprocess.check_output(['python3', 'setup.py', 'build_ext', '--inplace'], cwd='./cython_lwan_coro/', env=env)
source = ''
failure_count = 0
try:
sys.path.append('./cython_lwan_coro/')
wrapper = importlib.import_module('wrapper')
except ImportError as e:
failure_count = 1
source = str(e)
print(source) # Will be captured by check_output
import io
import subprocess
import importlib
import sys
import build_extension
def run(env, python_path):
# build the extension
# build_extension.build_ext(env)
source = subprocess.check_output([python_path + " -c 'import wrapper; wrapper.main()'"],
# cwd='./cython_lwan_coro/',
env=env,
shell=True,
)
failure_count = 0
expected_result = b'2\n' * 5 + b'3\n' * 5 + b'4\n' * 5
if source != expected_result:
failure_count = 1
result_dict = {'failed': failure_count, 'stdout': source}
return result_dict
#cython: language_level = 3
from cython.parallel import parallel
from libc.stdio cimport printf
"""
GOAL: implement nogil option in cdef class (extension types)
and native memory manager (refcount based) that does not
depend on cpython's memory manager and that does not require GIL.
HINT: look at C++ standard library (that works very nicely with Cython)
Cython documentation if here: http://docs.cython.org/en/latest/
Basic usage: http://docs.cython.org/en/latest/src/quickstart/build.html
Tutorial: http://docs.cython.org/en/latest/src/tutorial/cython_tutorial.html
Language: http://docs.cython.org/en/latest/src/userguide/language_basics.html
Extension types: http://docs.cython.org/en/latest/src/userguide/extension_types.html
Extension Types are the "pure cython" classes that I want to be able to
use without depending on cpython GIL (and in essence runtime, memory manager, etc.)
Cython memory allocation: http://docs.cython.org/en/latest/src/tutorial/memory_allocation.html
Parallelism: http://docs.cython.org/en/latest/src/userguide/parallelism.html
Explains how nogil is posisble in cython for anything that
only relies on C libraries that are multi-threaded
"""
# cdef class SomeMemory:
cdef class SomeMemory nogil:
"""
This is a cdef class which is also called
a extensino type. It is a kind of C struct
that also acts as a python object.
We would like to be able to define "nogil"
extension types:
cdef class SomeMemory nogil:
where all methods are "nogil" and memory
allocation does not depend on python runtime
"""
cdef int a;
cdef double b;
cdef void cinit(self, a, b) nogil:
# self.a = a
# self.b = b
pass
cdef void cdealloc(self) nogil:
pass
cdef void foo1(self) nogil:
"""
It is possible to define native C/Cython methods
that release the GIL (cool...)
"""
self.a += 3
cdef void foo2(self) nogil:
"""
It is possible to define native C/Cython methods
that release the GIL (cool...)
"""
self.a = 42
# Not allowed to define pure Python function in the extension type with nogil option now
# since we want this extension type is CPython free
# def baz(self):
# """
# It is also possible to define standard python
# methods
# """
# pass
# cdef bar(): # it is currently impossible to release GIL
cdef double bar1() nogil: # yet this is what we would like to
"""
This is a pure "cython method" which we would like to
be able to declare with nogil option but this requires
to first introduce the concept of nogil in cdef class
"""
cdef SomeMemory o1 = SomeMemory(1, 1.0) # Call init and get an object
o1.foo1()
return o1.a
cdef double bar2() nogil: # yet this is what we would like to
cdef SomeMemory o2 = SomeMemory(2, 2.0)
o2.foo2()
return o2.a
cpdef baz1():
"""
This method is both callable from python and pure "cython".
It can call both cdef methods and usual python functions
"""
return bar1()
cpdef baz2():
"""
This method is both callable from python and pure "cython".
It can call both cdef methods and usual python functions
"""
return bar2()
def bag():
return str(baz1()) + '\n' + str(baz2())
from distutils.core import setup
from distutils.extension import Extension
from Cython.Build import cythonize
setup(
ext_modules = cythonize([
Extension(
'nogil_extension',
['nogil_extension.pyx'],
),
])
)
from distutils.core import setup
from Cython.Build import cythonize
setup(
ext_modules = cythonize("test.pyx")
)
\ No newline at end of file
#cython: language_level = 3
from libc.stdio cimport printf
"""
GOAL: implement nogil option in cdef class (extension types)
and native memory manager (refcount based) that does not
depend on cpython's memory manager and that does not require GIL.
HINT: look at C++ standard library (that works very nicely with Cython)
Cython documentation if here: http://docs.cython.org/en/latest/
Basic usage: http://docs.cython.org/en/latest/src/quickstart/build.html
Tutorial: http://docs.cython.org/en/latest/src/tutorial/cython_tutorial.html
Language: http://docs.cython.org/en/latest/src/userguide/language_basics.html
Extension types: http://docs.cython.org/en/latest/src/userguide/extension_types.html
Extension Types are the "pure cython" classes that I want to be able to
use without depending on cpython GIL (and in essence runtime, memory manager, etc.)
Cython memory allocation: http://docs.cython.org/en/latest/src/tutorial/memory_allocation.html
Parallelism: http://docs.cython.org/en/latest/src/userguide/parallelism.html
Explains how nogil is posisble in cython for anything that
only relies on C libraries that are multi-threaded
"""
# cdef class SomeMemory:
cdef class SomeMemory nogil:
"""
This is a cdef class which is also called
a extensino type. It is a kind of C struct
that also acts as a python object.
We would like to be able to define "nogil"
extension types:
cdef class SomeMemory nogil:
where all methods are "nogil" and memory
allocation does not depend on python runtime
"""
cdef double a;
cdef double b;
cdef void foo(self) nogil:
"""
It is possible to define native C/Cython methods
that release the GIL (cool...)
"""
self.a = self.b
cdef void foo1(self, int a) nogil:
"""
It is possible to define native C/Cython methods
that release the GIL (cool...)
"""
self.a = a
cdef void foo3(self) nogil:
"""
It is possible to define native C/Cython methods
that release the GIL (cool...)
"""
pass
# Not allowed to define pure Python function in the extension type with nogil option now
# since we want this extension type is CPython free
# def baz(self):
# """
# It is also possible to define standard python
# methods
# """
# pass
# cdef bar(): # it is currently impossible to release GIL
cdef int bar() nogil: # yet this is what we would like to
"""
This is a pure "cython method" which we would like to
be able to declare with nogil option but this requires
to first introduce the concept of nogil in cdef class
"""
cdef SomeMemory o = SomeMemory(42.0, 3.14) # for this we need class allocation to handle memory without libpython
o.foo() # and we need method selection to be independent of libpython
o.foo1(2)
o.a = 1.0
return 0
cpdef baz():
"""
This method is both callable from python and pure "cython".
It can call both cdef methods and usual python functions
"""
bar()
# We call here a cpdef function, which calls a def function
# which then allocates cdef class SomeMemory
baz()
print("done")
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment