Clean up directives to distinguish between those that belong to a function or...

Clean up directives to distinguish between those that belong to a function or class and those that are generally inherited. Everything that is not inherited should also not have a default value and instead exist or not.
Then, prevent lambdas and generator expressions from inheriting directives from their outer function/scope, e.g. "@cython.cdef".
Closes #459.

CHANGES.rst

@@ -102,6 +102,9 @@ Bugs fixed
 * Fix declarations of builtin or C types using strings in pure python mode.
   (Github issue #2046)
 
+* Generator expressions and lambdas failed to compile in ``@cfunc`` functions.
+  (Github issue #459)
+
 * Several internal function signatures were fixed that lead to warnings in gcc-8.
   (Github issue #2363)
 

Cython/Compiler/Nodes.py

@@ -2340,7 +2340,7 @@ class CFuncDefNode(FuncDefNode):
         self.is_c_class_method = env.is_c_class_scope
         if self.directive_locals is None:
             self.directive_locals = {}
-        self.directive_locals.update(env.directives['locals'])
+        self.directive_locals.update(env.directives.get('locals', {}))
         if self.directive_returns is not None:
             base_type = self.directive_returns.analyse_as_type(env)
             if base_type is None:
@@ -2907,7 +2907,7 @@ class DefNode(FuncDefNode):
         self.py_wrapper.analyse_declarations(env)
 
     def analyse_argument_types(self, env):
-        self.directive_locals = env.directives['locals']
+        self.directive_locals = env.directives.get('locals', {})
         allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
 
         f2s = env.fused_to_specific
@@ -4959,7 +4959,7 @@ class CClassDefNode(ClassDefNode):
                     code.putln("if (__Pyx_MergeVtables(&%s) < 0) %s" % (
                         typeobj_cname,
                         code.error_goto(entry.pos)))
-            if not type.scope.is_internal and not type.scope.directives['internal']:
+            if not type.scope.is_internal and not type.scope.directives.get('internal'):
                 # scope.is_internal is set for types defined by
                 # Cython (such as closures), the 'internal'
                 # directive is set by users

Cython/Compiler/Options.py

@@ -170,8 +170,6 @@ _directive_defaults = {
     'nonecheck' : False,
     'initializedcheck' : True,
     'embedsignature' : False,
-    'locals' : {},
-    'exceptval' : None,  # (except value=None, check=True)
     'auto_cpdef': False,
     'auto_pickle': None,
     'cdivision': False,  # was True before 0.12
@@ -183,12 +181,8 @@ _directive_defaults = {
     'wraparound' : True,
     'ccomplex' : False,  # use C99/C++ for complex types and arith
     'callspec' : "",
-    'final' : False,
     'nogil' : False,
-    'internal' : False,
     'profile': False,
-    'no_gc_clear': False,
-    'no_gc': False,
     'linetrace': False,
     'emit_code_comments': True,  # copy original source code into C code comments
     'annotation_typing': True,  # read type declarations from Python function annotations
@@ -241,7 +235,6 @@ _directive_defaults = {
 
 # experimental, subject to change
     'binding': None,
-    'freelist': 0,
 
     'formal_grammar': False,
 }
@@ -301,7 +294,9 @@ def normalise_encoding_name(option_name, encoding):
 directive_types = {
     'language_level': int,  # values can be None/2/3, where None == 2+warning
     'auto_pickle': bool,
+    'locals': dict,
     'final' : bool,  # final cdef classes and methods
+    'nogil' : bool,
     'internal' : bool,  # cdef class visibility in the module dict
     'infer_types' : bool,  # values can be True/None/False
     'binding' : bool,
@@ -310,7 +305,10 @@ directive_types = {
     'inline' : None,
     'staticmethod' : None,
     'cclass' : None,
+    'no_gc_clear' : bool,
+    'no_gc' : bool,
     'returns' : type,
+    'exceptval': type,  # actually (type, check=True/False), but has its own parser
     'set_initial_path': str,
     'freelist': int,
     'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'),
@@ -325,8 +323,10 @@ directive_scopes = {  # defaults to available everywhere
     # 'module', 'function', 'class', 'with statement'
     'auto_pickle': ('module', 'cclass'),
     'final' : ('cclass', 'function'),
-    'nogil' : ('function',),
+    'nogil' : ('function', 'with statement'),
     'inline' : ('function',),
+    'cfunc' : ('function', 'with statement'),
+    'ccall' : ('function', 'with statement'),
     'returns' : ('function',),
     'exceptval' : ('function',),
     'locals' : ('function',),
@@ -334,6 +334,7 @@ directive_scopes = {  # defaults to available everywhere
     'no_gc_clear' : ('cclass',),
     'no_gc' : ('cclass',),
     'internal' : ('cclass',),
+    'cclass' : ('class', 'cclass', 'with statement'),
     'autotestdict' : ('module',),
     'autotestdict.all' : ('module',),
     'autotestdict.cdef' : ('module',),

Cython/Compiler/ParseTreeTransforms.py

@@ -958,30 +958,33 @@ class InterpretCompilerDirectives(CythonTransform):
         else:
             assert False
 
-    def visit_with_directives(self, body, directives):
-        olddirectives = self.directives
-        newdirectives = copy.copy(olddirectives)
-        newdirectives.update(directives)
-        self.directives = newdirectives
-        assert isinstance(body, Nodes.StatListNode), body
-        retbody = self.visit_Node(body)
-        directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
-                                                 directives=newdirectives)
-        self.directives = olddirectives
-        return directive
+    def visit_with_directives(self, node, directives):
+        if not directives:
+            return self.visit_Node(node)
+
+        old_directives = self.directives
+        new_directives = dict(old_directives)
+        new_directives.update(directives)
+
+        if new_directives == old_directives:
+            return self.visit_Node(node)
+
+        self.directives = new_directives
+        retbody = self.visit_Node(node)
+        self.directives = old_directives
+
+        if not isinstance(retbody, Nodes.StatListNode):
+            retbody = Nodes.StatListNode(node.pos, stats=[retbody])
+        return Nodes.CompilerDirectivesNode(
+            pos=retbody.pos, body=retbody, directives=new_directives)
 
     # Handle decorators
     def visit_FuncDefNode(self, node):
         directives = self._extract_directives(node, 'function')
-        if not directives:
-            return self.visit_Node(node)
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def visit_CVarDefNode(self, node):
         directives = self._extract_directives(node, 'function')
-        if not directives:
-            return self.visit_Node(node)
         for name, value in directives.items():
             if name == 'locals':
                 node.directive_locals = value
@@ -990,29 +993,19 @@ class InterpretCompilerDirectives(CythonTransform):
                     node.pos,
                     "Cdef functions can only take cython.locals(), "
                     "staticmethod, or final decorators, got %s." % name))
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def visit_CClassDefNode(self, node):
         directives = self._extract_directives(node, 'cclass')
-        if not directives:
-            return self.visit_Node(node)
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def visit_CppClassNode(self, node):
         directives = self._extract_directives(node, 'cppclass')
-        if not directives:
-            return self.visit_Node(node)
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def visit_PyClassDefNode(self, node):
         directives = self._extract_directives(node, 'class')
-        if not directives:
-            return self.visit_Node(node)
-        body = Nodes.StatListNode(node.pos, stats=[node])
-        return self.visit_with_directives(body, directives)
+        return self.visit_with_directives(node, directives)
 
     def _extract_directives(self, node, scope_name):
         if not node.decorators:
@@ -1059,7 +1052,7 @@ class InterpretCompilerDirectives(CythonTransform):
                 optdict[name] = value
         return optdict
 
-    # Handle with statements
+    # Handle with-statements
     def visit_WithStatNode(self, node):
         directive_dict = {}
         for directive in self.try_to_parse_directives(node.manager) or []:
@@ -2385,6 +2378,10 @@ class AdjustDefByDirectives(CythonTransform, SkipDeclarations):
         self.visitchildren(node)
         return node
 
+    def visit_LambdaNode(self, node):
+        # No directives should modify lambdas or generator expressions (and also nothing in them).
+        return node
+
     def visit_PyClassDefNode(self, node):
         if 'cclass' in self.directives:
             node = node.as_cclass()

tests/run/exttype_freelist.pyx

@@ -451,3 +451,19 @@ cdef class ExtTypeWithRefCycle:
 
     def __init__(self, obj=None):
         self.attribute = obj
+
+
+@cython.freelist(3)
+@cython.cclass
+class DecoratedPyClass(object):
+    """
+    >>> obj1 = DecoratedPyClass()
+    >>> obj2 = DecoratedPyClass()
+    >>> obj3 = DecoratedPyClass()
+    >>> obj4 = DecoratedPyClass()
+
+    >>> obj1 = DecoratedPyClass()
+    >>> obj2 = DecoratedPyClass()
+    >>> obj3 = DecoratedPyClass()
+    >>> obj4 = DecoratedPyClass()
+    """

tests/run/purecdef.py

@@ -28,6 +28,17 @@ with cfunc:
     def fwith2(a):
         return a*4
 
+    @cython.test_assert_path_exists(
+        '//CFuncDefNode',
+        '//LambdaNode',
+        '//GeneratorDefNode',
+        '//GeneratorBodyDefNode',
+    )
+    def f_with_genexpr(a):
+        f = lambda x: x+1
+        return (f(x) for x in a)
+
+
 with cclass:
     @cython.test_assert_path_exists('//CClassDefNode')
     class Egg(object):
@@ -123,3 +134,14 @@ def test_typed_return():
     """
     x = cython.declare(int, 5)
     assert typed_return(cython.address(x))[0] is x
+
+
+def test_genexpr_in_cdef(l):
+    """
+    >>> gen = test_genexpr_in_cdef([1, 2, 3])
+    >>> list(gen)
+    [2, 3, 4]
+    >>> list(gen)
+    []
+    """
+    return f_with_genexpr(l)