Commit ab8b5f9d authored by Mark Florisson's avatar Mark Florisson

Cython Utility Code cname extclass decorator + extmethod prototypes

parent 0daf7250
......@@ -63,20 +63,14 @@ class CythonScope(ModuleScope):
defining = 1,
cname = 'PyObject_TypeCheck')
self.test_cythonscope()
# self.test_cythonscope()
def test_cythonscope(self):
# A special function just to make it easy to test the scope and
# utility code functionality in isolation. It is available to
# "end-users" but nobody will know it is there anyway...
entry = self.declare_cfunction(
'_testscope',
CFuncType(py_object_type, [CFuncTypeArg("value", c_int_type, None)]),
pos=None,
defining=1,
cname='__pyx_testscope'
)
entry.utility_code_definition = cython_testscope_utility_code
cython_testscope_utility_code.declare_in_scope(self)
cython_test_extclass_utility_code.declare_in_scope(self)
#
# The view sub-scope
......@@ -85,29 +79,83 @@ class CythonScope(ModuleScope):
self.declare_module('view', viewscope, None)
viewscope.is_cython_builtin = True
viewscope.pxd_file_loaded = True
entry = viewscope.declare_cfunction(
'_testscope',
CFuncType(py_object_type, [CFuncTypeArg("value", c_int_type, None)]),
pos=None,
defining=1,
cname='__pyx_view_testscope'
)
entry.utility_code_definition = cythonview_testscope_utility_code
def create_cython_scope(context):
cythonview_testscope_utility_code.declare_in_scope(viewscope)
def create_cython_scope(context, create_testscope):
# One could in fact probably make it a singleton,
# but not sure yet whether any code mutates it (which would kill reusing
# it across different contexts)
return CythonScope()
scope = CythonScope()
if create_testscope:
scope.test_cythonscope()
return scope
cython_testscope_utility_code = CythonUtilityCode(u"""
@cname('__pyx_testscope')
cdef object _testscope(int value):
return "hello from cython scope, value=%d" % value
""")# #, name="cython utility code", prefix="__pyx_cython_")
""")
undecorated_methods_protos = UtilityCode(proto=u"""
/* These methods are undecorated and have therefore no prototype */
static PyObject *__pyx_TestClass_cdef_method(
struct __pyx_TestClass *self, int value);
static PyObject *__pyx_TestClass_cpdef_method(
struct __pyx_TestClass *self, int value, int skip_dispatch);
static PyObject *__pyx_TestClass_def_method(
PyObject *self, PyObject *value);
""")
cython_test_extclass_utility_code = CythonUtilityCode(
name="TestClassUtilityCode",
prefix="__pyx_prefix_TestClass_",
requires=[undecorated_methods_protos],
impl=u"""
@cname('__pyx_TestClass')
cdef class TestClass(object):
cdef public int value
def __init__(self, int value):
self.value = value
def __str__(self):
return 'TestClass(%d)' % self.value
cdef cdef_method(self, int value):
print 'Hello from cdef_method', value
cpdef cpdef_method(self, int value):
print 'Hello from cpdef_method', value
def def_method(self, int value):
print 'Hello from def_method', value
@cname('cdef_cname')
cdef cdef_cname_method(self, int value):
print "Hello from cdef_cname_method", value
@cname('cpdef_cname')
cpdef cpdef_cname_method(self, int value):
print "Hello from cpdef_cname_method", value
@cname('def_cname')
def def_cname_method(self, int value):
print "Hello from def_cname_method", value
@cname('__pyx_TestClass_New')
cdef _testclass_new(int value):
return TestClass(value)
""")
cythonview_testscope_utility_code = CythonUtilityCode(u"""
@cname('__pyx_view_testscope')
cdef object _testscope(int value):
return "hello from cython.view scope, value=%d" % value
""") #, name="cython utility code", prefix="__pyx_cython_view_")
""")
......@@ -3615,10 +3615,13 @@ class AttributeNode(ExprNode):
def analyse_types(self, env, target = 0):
if self.analyse_as_cimported_attribute(env, target):
return
if not target and self.analyse_as_unbound_cmethod(env):
return
self.analyse_as_ordinary_attribute(env, target)
self.entry.used = True
elif not target and self.analyse_as_unbound_cmethod(env):
self.entry.used = True
else:
self.analyse_as_ordinary_attribute(env, target)
if self.entry:
self.entry.used = True
def analyse_as_cimported_attribute(self, env, target):
# Try to interpret this as a reference to an imported
......
......@@ -63,14 +63,17 @@ class Context(object):
cython_scope = None
def __init__(self, include_directories, compiler_directives, cpp=False, language_level=2, options=None):
def __init__(self, include_directories, compiler_directives, cpp=False,
language_level=2, options=None, create_testscope=True):
# cython_scope is a hack, set to False by subclasses, in order to break
# an infinite loop.
# Better code organization would fix it.
import Builtin, CythonScope
self.modules = {"__builtin__" : Builtin.builtin_scope}
self.modules["cython"] = self.cython_scope = CythonScope.create_cython_scope(self)
cyscope = CythonScope.create_cython_scope(
self, create_testscope=create_testscope)
self.modules["cython"] = self.cython_scope = cyscope
self.include_directories = include_directories
self.future_directives = set()
self.compiler_directives = compiler_directives
......
......@@ -951,6 +951,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
# code.putln("static PyTypeObject %s;" % name)
def generate_exttype_vtable_struct(self, entry, code):
if not entry.used:
return
code.mark_pos(entry.pos)
# Generate struct declaration for an extension type's vtable.
type = entry.type
......@@ -967,11 +970,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
for method_entry in scope.cfunc_entries:
if not method_entry.is_inherited:
code.putln(
"%s;" % method_entry.type.declaration_code("(*%s)" % method_entry.name))
"%s;" % method_entry.type.declaration_code("(*%s)" % method_entry.cname))
code.putln(
"};")
def generate_exttype_vtabptr_declaration(self, entry, code):
if not entry.used:
return
code.mark_pos(entry.pos)
# Generate declaration of pointer to an extension type's vtable.
type = entry.type
......@@ -1078,37 +1084,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
def generate_cfunction_declarations(self, env, code, definition):
for entry in env.cfunc_entries:
if entry.inline_func_in_pxd or (not entry.in_cinclude and (definition
or entry.defined_in_pxd or entry.visibility == 'extern')):
if entry.visibility == 'extern':
storage_class = "%s " % Naming.extern_c_macro
dll_linkage = "DL_IMPORT"
elif entry.visibility == 'public':
storage_class = "%s " % Naming.extern_c_macro
dll_linkage = "DL_EXPORT"
elif entry.visibility == 'private':
storage_class = "static "
dll_linkage = None
else:
storage_class = "static "
dll_linkage = None
type = entry.type
if entry.defined_in_pxd and not definition:
storage_class = "static "
dll_linkage = None
type = CPtrType(type)
header = type.declaration_code(entry.cname,
dll_linkage = dll_linkage)
if entry.func_modifiers:
modifiers = "%s " % ' '.join(entry.func_modifiers).upper()
else:
modifiers = ''
code.putln("%s%s%s; /*proto*/" % (
storage_class,
modifiers,
header))
generate_cfunction_declaration(entry, env, code, definition)
def generate_variable_definitions(self, env, code):
for entry in env.var_entries:
......@@ -2413,6 +2389,39 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
"%s = &%s;" % (
type.typeptr_cname, type.typeobj_cname))
def generate_cfunction_declaration(entry, env, code, definition):
if entry.inline_func_in_pxd or (not entry.in_cinclude and (definition
or entry.defined_in_pxd or entry.visibility == 'extern')):
if entry.visibility == 'extern':
storage_class = "%s " % Naming.extern_c_macro
dll_linkage = "DL_IMPORT"
elif entry.visibility == 'public':
storage_class = "%s " % Naming.extern_c_macro
dll_linkage = "DL_EXPORT"
elif entry.visibility == 'private':
storage_class = "static "
dll_linkage = None
else:
storage_class = "static "
dll_linkage = None
type = entry.type
if entry.defined_in_pxd and not definition:
storage_class = "static "
dll_linkage = None
type = CPtrType(type)
header = type.declaration_code(entry.cname,
dll_linkage = dll_linkage)
if entry.func_modifiers:
modifiers = "%s " % ' '.join(entry.func_modifiers).upper()
else:
modifiers = ''
code.putln("%s%s%s; /*proto*/" % (
storage_class,
modifiers,
header))
#------------------------------------------------------------------------------------
#
# Runtime support code
......
......@@ -3473,6 +3473,7 @@ class CClassDefNode(ClassDefNode):
api = self.api,
buffer_defaults = self.buffer_defaults(env),
shadow = self.shadow)
if self.shadow:
home_scope.lookup(self.class_name).as_variable = self.entry
if home_scope is not env and self.visibility == 'extern':
......@@ -6728,6 +6729,8 @@ class CnameDecoratorNode(StatNode):
cdef func(...):
...
In case of a cdef class the cname specifies the objstruct_cname.
node the node to which the cname decorator is applied
cname the cname the node should get
"""
......@@ -6736,8 +6739,58 @@ class CnameDecoratorNode(StatNode):
def analyse_declarations(self, env):
self.node.analyse_declarations(env)
self.node.entry.cname = self.cname
self.node.entry.func_cname = self.cname
self.is_function = isinstance(self.node, FuncDefNode)
e = self.node.entry
if self.is_function:
e.cname = self.cname
e.func_cname = self.cname
else:
scope = self.node.scope
e.cname = self.cname
e.type.objstruct_cname = self.cname
e.type.typeobj_cname = Naming.typeobj_prefix + self.cname
e.type.typeptr_cname = Naming.typeptr_prefix + self.cname
e.as_variable.cname = py_object_type.cast_code(e.type.typeptr_cname)
scope.scope_prefix = self.cname + "_"
for name, entry in scope.entries.iteritems():
if entry.func_cname:
entry.func_cname = '%s_%s' % (self.cname, entry.cname)
def analyse_expressions(self, env):
self.node.analyse_expressions(env)
def generate_function_definitions(self, env, code):
if self.is_function and env.is_c_class_scope:
# method in cdef class, generate a prototype in the header
h_code = code.globalstate['utility_code_proto']
if isinstance(self.node, DefNode):
self.node.generate_function_header(
h_code, with_pymethdef=False, proto_only=True)
else:
import ModuleNode
entry = self.node.entry
cname = entry.cname
entry.cname = entry.func_cname
ModuleNode.generate_cfunction_declaration(
entry,
env.global_scope(),
h_code,
definition=True)
entry.cname = cname
self.node.generate_function_definitions(env, code)
def generate_execution_code(self, code):
self.node.generate_execution_code(code)
#------------------------------------------------------------------------------------
......
......@@ -1250,6 +1250,9 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
"""
def handle_function(self, node):
if not node.decorators:
return self.visit_Node(node)
for i, decorator in enumerate(node.decorators):
decorator = decorator.decorator
......@@ -1271,18 +1274,16 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
raise AssertionError(
"argument to cname decorator must be a string literal")
cname = args[0].compile_time_value(None)
cname = args[0].compile_time_value(None).decode('UTF-8')
del node.decorators[i]
node = Nodes.CnameDecoratorNode(pos=node.pos, node=node,
cname=cname)
break
self.visitchildren(node)
return node
return self.visit_Node(node)
visit_CFuncDefNode = handle_function
# visit_FuncDefNode = handle_function
# visit_ClassDefNode = handle_function
visit_FuncDefNode = handle_function
visit_CClassDefNode = handle_function
class ForwardDeclareTypes(CythonTransform):
......@@ -1570,10 +1571,6 @@ if VALUE is not None:
self.visitchildren(node)
return None
def visit_CnameDecoratorNode(self, node):
self.visitchildren(node)
return node.node
def create_Property(self, entry):
if entry.visibility == 'public':
if entry.type.is_pyobject:
......@@ -2567,4 +2564,3 @@ class DebugTransform(CythonTransform):
self.tb.start('LocalVar', attrs)
self.tb.end('LocalVar')
......@@ -64,6 +64,8 @@ def use_utility_code_definitions(scope, target):
for entry in scope.entries.itervalues():
if entry.used and entry.utility_code_definition:
target.use_utility_code(entry.utility_code_definition)
for required_utility in entry.utility_code_definition.requires:
target.use_utility_code(required_utility)
elif entry.as_module:
use_utility_code_definitions(entry.as_module, target)
......
......@@ -242,8 +242,10 @@ class StringSourceDescriptor(SourceDescriptor):
def get_filenametable_entry(self):
return "stringsource"
def __hash__(self):
return hash(self.name)
# Do not hash on the name, an identical string source should be the
# same object (name is often defaulted in other places)
# def __hash__(self):
# return hash(self.name)
def __eq__(self, other):
return isinstance(other, StringSourceDescriptor) and self.name == other.name
......
......@@ -178,6 +178,7 @@ class Entry(object):
might_overflow = 0
utility_code_definition = None
in_with_gil_block = 0
from_cython_utility_code = None
def __init__(self, name, cname, type, pos = None, init = None):
self.name = name
......@@ -200,6 +201,7 @@ class Entry(object):
def all_alternatives(self):
return [self] + self.overloaded_alternatives
class Scope(object):
# name string Unqualified name
# outer_scope Scope or None Enclosing scope
......@@ -278,10 +280,14 @@ class Scope(object):
self.return_type = None
self.id_counters = {}
def merge_in(self, other):
def merge_in(self, other, merge_unused=True):
# Use with care...
self.entries.update(other.entries)
for x in ('const_entries',
entries = [(name, entry)
for name, entry in other.entries.iteritems()
if entry.used or merge_unused]
self.entries.update(entries)
for attr in ('const_entries',
'type_entries',
'sue_entries',
'arg_entries',
......@@ -289,8 +295,10 @@ class Scope(object):
'pyfunc_entries',
'cfunc_entries',
'c_class_entries'):
getattr(self, x).extend(getattr(other, x))
self_entries = getattr(self, attr)
for entry in getattr(other, attr):
if entry.used or merge_unused:
self_entries.append(entry)
def __str__(self):
return "<%s %s>" % (self.__class__.__name__, self.qualified_name)
......@@ -1226,6 +1234,11 @@ class ModuleScope(Scope):
if type.typeobj_cname and type.typeobj_cname != typeobj_cname:
error(pos, "Type object name differs from previous declaration")
type.typeobj_cname = typeobj_cname
# cdef classes are always exported, but we need to set it to
# distinguish between unused Cython utility code extension classes
entry.used = True
#
# Return new or existing entry
#
......
......@@ -22,7 +22,8 @@ Support for parsing strings into code trees.
class StringParseContext(Main.Context):
def __init__(self, name, include_directories=None):
if include_directories is None: include_directories = []
Main.Context.__init__(self, include_directories, {})
Main.Context.__init__(self, include_directories, {},
create_testscope=False)
self.module_name = name
def find_module(self, module_name, relative_to = None, pos = None, need_pxd = 1):
......
......@@ -4,22 +4,68 @@ import Symtab
import Naming
from Cython.Compiler import Visitor
class NonManglingModuleScope(Symtab.ModuleScope):
def add_imported_entry(self, name, entry, pos):
entry.used = True
return super(NonManglingModuleScope, self).add_imported_entry(
name, entry, pos)
def mangle(self, prefix, name=None):
if name:
if prefix in (Naming.typeobj_prefix, Naming.func_prefix):
# Functions, classes etc. gets a manually defined prefix easily
# manually callable instead (the one passed to CythonUtilityCode)
prefix = self.prefix
return "%s%s" % (prefix, name)
else:
return self.base.name
class CythonUtilityCodeContext(StringParseContext):
scope = None
def find_module(self, module_name, relative_to = None, pos = None,
need_pxd = 1):
if module_name != self.module_name:
raise AssertionError("Not yet supporting any cimports/includes "
"from string code snippets")
if self.scope is None:
self.scope = NonManglingModuleScope(
module_name, parent_module=None, context=self)
self.scope.prefix = self.prefix
return self.scope
class CythonUtilityCode(object):
"""
Utility code written in the Cython language itself.
The @cname decorator can set the cname for a function, method of cdef class.
Functions decorated with @cname('c_func_name') get the given cname.
For cdef classes the rules are as follows:
obj struct -> <cname>
obj type ptr -> __pyx_ptype_<cname>
methods -> <class_cname>_<method_cname>
For methods the cname decorator is optional, but without the decorator the
methods will not be prototyped. See Cython.Compiler.CythonScope and
tests/run/cythonscope.pyx for examples.
"""
def __init__(self, pyx, name="<utility code>", prefix=""):
def __init__(self, impl, name="CythonUtilityCode", prefix="", requires=None):
# 1) We need to delay the parsing/processing, so that all modules can be
# imported without import loops
# 2) The same utility code object can be used for multiple source files;
# while the generated node trees can be altered in the compilation of a
# single file.
# Hence, delay any processing until later.
self.pyx = pyx
self.pyx = impl
self.name = name
self.prefix = prefix
self.requires = requires or []
def get_tree(self):
from AnalysedTreeTransforms import AutoTestDictTransform
......@@ -29,9 +75,9 @@ class CythonUtilityCode(object):
excludes = [AutoTestDictTransform]
import Pipeline, ParseTreeTransforms
#context = CythonUtilityCodeContext(self.name)
#context.prefix = self.prefix
context = StringParseContext(self.name)
context = CythonUtilityCodeContext(self.name)
context.prefix = self.prefix
#context = StringParseContext(self.name)
tree = parse_from_strings(self.name, self.pyx, context=context)
pipeline = Pipeline.create_pipeline(context, 'pyx', exclude_classes=excludes)
......@@ -43,8 +89,28 @@ class CythonUtilityCode(object):
before=before)
(err, tree) = Pipeline.run_pipeline(pipeline, tree)
assert not err
assert not err, err
return tree
def put_code(self, output):
pass
def declare_in_scope(self, dest_scope):
"""
Declare all entries from the utility code in dest_scope. Code will only
be included for used entries.
"""
self.tree = self.get_tree()
entries = self.tree.scope.entries
entries.pop('__name__')
entries.pop('__file__')
entries.pop('__builtins__')
entries.pop('__doc__')
for name, entry in entries.iteritems():
entry.utility_code_definition = self
entry.used = False
dest_scope.merge_in(self.tree.scope, merge_unused=True)
self.tree.scope = dest_scope
cimport cython
from cython cimport _testscope as tester
from cython cimport TestClass, _testclass_new as TestClass_New
from cython.view cimport _testscope as viewtester
cdef extern from *:
# TestClass stuff
cdef struct __pyx_TestClass:
int value
# Type pointer
cdef __pyx_TestClass *TestClassType "__pyx_ptype___pyx_TestClass"
# This is a cdef function
cdef __pyx_TestClass_New(int)
# These are methods and therefore have no prototypes
cdef __pyx_TestClass_cdef_method(TestClass self, int value)
cdef __pyx_TestClass_cpdef_method(TestClass self, int value, int skip_dispatch)
cdef __pyx_TestClass_def_method(object self, object value)
cdef __pyx_TestClass_cdef_cname(TestClass self, int value)
cdef __pyx_TestClass_cpdef_cname(TestClass self, int value, int skip_dispatch)
cdef __pyx_TestClass_def_cname(object self, object value)
def test_cdef_cython_utility():
"""
......@@ -16,3 +37,93 @@ def test_cdef_cython_utility():
print cython.view._testscope(4)
print tester(3)
print viewtester(3)
def test_cdef_class_cython_utility():
"""
>>> test_cdef_class_cython_utility()
7
14
TestClass(20)
TestClass(50)
"""
cdef __pyx_TestClass *objstruct
obj = TestClass_New(7)
objstruct = <__pyx_TestClass *> obj
print objstruct.value
obj = __pyx_TestClass_New(14)
objstruct = <__pyx_TestClass *> obj
print objstruct.value
print (<object> TestClassType)(20)
print TestClass(50)
def test_extclass_c_methods():
"""
>>> test_extclass_c_methods()
Hello from cdef_method 1
Hello from cpdef_method 2
Hello from def_method 3
Hello from cdef_cname_method 4
Hello from cpdef_cname_method 5
Hello from def_cname_method 6
Hello from cdef_method 1
Hello from cpdef_method 2
Hello from def_method 3
Hello from cdef_cname_method 4
Hello from cpdef_cname_method 5
Hello from def_cname_method 6
"""
cdef TestClass obj1 = TestClass(11)
cdef TestClass obj2 = TestClass_New(22)
__pyx_TestClass_cdef_method(obj1, 1)
__pyx_TestClass_cpdef_method(obj1, 2, True)
__pyx_TestClass_def_method(obj1, 3)
__pyx_TestClass_cdef_cname(obj1, 4)
__pyx_TestClass_cpdef_cname(obj1, 5, True)
__pyx_TestClass_def_cname(obj1, 6)
__pyx_TestClass_cdef_method(obj2, 1)
__pyx_TestClass_cpdef_method(obj2, 2, True)
__pyx_TestClass_def_method(obj2, 3)
__pyx_TestClass_cdef_cname(obj2, 4)
__pyx_TestClass_cpdef_cname(obj2, 5, True)
__pyx_TestClass_def_cname(obj2, 6)
def test_extclass_cython_methods():
"""
>>> test_extclass_cython_methods()
Hello from cdef_method 1
Hello from cpdef_method 2
Hello from def_method 3
Hello from cdef_cname_method 4
Hello from cpdef_cname_method 5
Hello from def_cname_method 6
Hello from cdef_method 1
Hello from cpdef_method 2
Hello from def_method 3
Hello from cdef_cname_method 4
Hello from cpdef_cname_method 5
Hello from def_cname_method 6
"""
cdef TestClass obj1 = TestClass(11)
cdef TestClass obj2 = TestClass_New(22)
obj1.cdef_method(1)
obj1.cpdef_method(2)
obj1.def_method(3)
obj1.cdef_cname_method(4)
obj1.cpdef_cname_method(5)
obj1.def_cname_method(6)
obj2.cdef_method(1)
obj2.cpdef_method(2)
obj2.def_method(3)
obj2.cdef_cname_method(4)
obj2.cpdef_cname_method(5)
obj2.def_cname_method(6)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment