Change annotation typing directive to cover all type annotations and modify...

Change annotation typing directive to cover all type annotations and modify annotation parsing to be more PEP 484 compatible.
Only "cython.*" types are interpreted as C types from now on, everything else is considered Python types for maximum compatibility with Python type hints.

CHANGES.rst

@@ -22,8 +22,13 @@ Features added
   https://www.python.org/dev/peps/pep-0525/
   https://www.python.org/dev/peps/pep-0530/
 
-* Annotations are now included in the signature docstring generated by the
-  ``embedsignature`` directive.  Patch by Lisandro Dalcin (Github issue #1781).
+* Variable annotations are now parsed according to PEP 526.  Cython types (e.g.
+  ``cython.int``) are evaluated as C type declarations and everything else as Python
+  types.  This can be disabled with the directive ``annotation_typing=False``.
+  (Github issue #1850)
+
+* Signature annotations are now included in the signature docstring generated by
+  the ``embedsignature`` directive.  Patch by Lisandro Dalcin (Github issue #1781).
 
 * ``len(memoryview)`` can be used in nogil sections to get the size of the
   first dimension of a memory view (``shape[0]``). (Github issue #1733)
@@ -62,6 +67,14 @@ Bugs fixed
 Other changes
 -------------
 
+* Type declarations in signature annotations are now parsed according to PEP 484
+  typing.  Only Cython types (e.g. ``cython.int``) and Python builtin types are
+  currently considered as type declarations.  Everything else is ignored, but this
+  will probably change in a future Cython release.  (Github issue #1672)
+
+* The directive ``annotation_typing`` is now ``True`` by default, which enables
+  parsing type declarations from annotations.
+
 * This release no longer supports Python 3.2.
 
 

Cython/Compiler/ExprNodes.py

@@ -27,7 +27,7 @@ from .Code import UtilityCode, TempitaUtilityCode
 from . import StringEncoding
 from . import Naming
 from . import Nodes
-from .Nodes import Node, utility_code_for_imports
+from .Nodes import Node, utility_code_for_imports, analyse_type_annotation
 from . import PyrexTypes
 from .PyrexTypes import py_object_type, c_long_type, typecast, error_type, \
     unspecified_type
@@ -1839,6 +1839,8 @@ class NameNode(AtomicExprNode):
         String literals are allowed and ignored.
         The ambiguous Python types 'int' and 'long' are ignored and the 'cython.int' form must be used instead.
         """
+        if not env.directives['annotation_typing']:
+            return
         if env.is_module_scope or env.is_py_class_scope:
             # annotations never create global cdef names and Python classes don't support them anyway
             return
@@ -1848,18 +1850,11 @@ class NameNode(AtomicExprNode):
             return
 
         annotation = self.annotation
-        atype = annotation.analyse_as_type(env)
-        if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'):
-            # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations
-            if atype in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type):
-                atype = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type
-        elif annotation.is_string_literal:
+        if annotation.is_string_literal:
             # name: "description" => not a type, but still a declared variable or attribute
             atype = None
-        elif atype is None:
-            # annotations always make variables local => ignore and leave to type inference
-            warning(annotation.pos, "Unknown type declaration in annotation, ignoring")
-
+        else:
+            _, atype = analyse_type_annotation(annotation, env)
         if atype is None:
             atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type
         self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target)

Cython/Compiler/Nodes.py

@@ -68,10 +68,12 @@ def embed_position(pos, docstring):
     return doc
 
 
-def _analyse_signature_annotation(annotation, env):
+def analyse_type_annotation(annotation, env):
     base_type = None
     explicit_pytype = explicit_ctype = False
     if annotation.is_dict_literal:
+        warning(annotation.pos,
+                "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.")
         for name, value in annotation.key_value_pairs:
             if not name.is_string_literal:
                 continue
@@ -85,6 +87,13 @@ def _analyse_signature_annotation(annotation, env):
         if explicit_pytype and explicit_ctype:
             warning(annotation.pos, "Duplicate type declarations found in signature annotation")
     arg_type = annotation.analyse_as_type(env)
+    if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'):
+        # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations
+        if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type):
+            arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type
+    elif arg_type is not None and annotation.is_string_literal:
+        warning(annotation.pos,
+                "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.")
     if arg_type is not None:
         if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject:
             warning(annotation.pos,
@@ -92,7 +101,7 @@ def _analyse_signature_annotation(annotation, env):
         base_type = CAnalysedBaseTypeNode(
             annotation.pos, type=arg_type, is_arg=True)
     else:
-        warning(annotation.pos, "Unknown type declaration found in signature annotation")
+        warning(annotation.pos, "Unknown type declaration in annotation, ignoring")
     return base_type, arg_type
 
 
@@ -870,7 +879,7 @@ class CArgDeclNode(Node):
         annotation = self.annotation
         if not annotation:
             return None
-        base_type, arg_type = _analyse_signature_annotation(annotation, env)
+        base_type, arg_type = analyse_type_annotation(annotation, env)
         if base_type is not None:
             self.base_type = base_type
         return arg_type
@@ -2796,7 +2805,7 @@ class DefNode(FuncDefNode):
         # if a signature annotation provides a more specific return object type, use it
         if self.return_type is py_object_type and self.return_type_annotation:
             if env.directives['annotation_typing'] and not self.entry.is_special:
-                _, return_type = _analyse_signature_annotation(self.return_type_annotation, env)
+                _, return_type = analyse_type_annotation(self.return_type_annotation, env)
                 if return_type and return_type.is_pyobject:
                     self.return_type = return_type
 

Cython/Compiler/Options.py

@@ -161,7 +161,7 @@ _directive_defaults = {
     'no_gc': False,
     'linetrace': False,
     'emit_code_comments': True,  # copy original source code into C code comments
-    'annotation_typing': False,  # read type declarations from Python function annotations
+    'annotation_typing': True,   # read type declarations from Python function annotations
     'infer_types': None,
     'infer_types.verbose': False,
     'autotestdict': True,

docs/src/tutorial/pure.rst

@@ -195,22 +195,40 @@ Static typing
 
 * ``@cython.returns(<type>)`` specifies the function's return type.
 
-* Starting with Cython 0.21, Python signature annotations can be used to
-  declare argument types.  Cython recognises three ways to do this, as
-  shown in the following example.  Note that it currently needs to be
-  enabled explicitly with the directive ``annotation_typing=True``.
-  This might change in a later version.
+* Python annotations can be used to declare argument types, as shown in the
+  following example.  To avoid conflicts with other kinds of annotation
+  usages, this can be disabled with the directive ``annotation_typing=False``.
 
   ::
 
-    # cython: annotation_typing=True
-
-    def func(plain_python_type: dict,
-             named_python_type: 'dict',
-             explicit_python_type: {'type': dict},
-             explicit_c_type: {'ctype': 'int'}):
+    def func(a_pydict: dict, a_cint: cython.int) -> tuple:
         ...
 
+  Since version 0.27, Cython also supports the variable annotations defined
+  in `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_. This allows to
+  declare types of variables in a Python 3.6 compatible way as follows::
+
+    def func():
+        # Cython types are evaluated as for cdef declarations
+        x : cython.int               # cdef int x
+        y : cython.double = 0.57721  # cdef double y = 0.57721
+        z : cython.float  = 0.57721  # cdef float z  = 0.57721
+
+        # Python types shadow Cython types for compatibility reasons
+        a : float = 0.54321          # cdef double a = 0.54321
+        b : int = 5                  # cdef object b = 5
+        c : long = 6                 # cdef object c = 6
+
+    @cython.cclass
+    class A:
+        a : cython.int
+        b : cython.int
+        def __init__(self, b=0):
+            self.a = 3
+            self.b = b
+
+  There is currently no way to express the visibility of object attributes.
+
 
 C types
 ^^^^^^^

tests/run/annotation_typing.pyx

-# cython: annotation_typing=True
+# mode: run
+# tag: pep484, warnings
 
+cimport cython
 from cython cimport typeof
 
 
-def pytypes_def(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4) -> list:
+def old_dict_syntax(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4) -> list:
     """
-    >>> pytypes_def([1])
+    >>> old_dict_syntax([1])
     ('list object', 'int', 'long', 'float')
     [1, 2, 3, 4.0]
-    >>> pytypes_def([1], 3)
+    >>> old_dict_syntax([1], 3)
     ('list object', 'int', 'long', 'float')
     [1, 3, 3, 4.0]
+    >>> old_dict_syntax(123)
+    Traceback (most recent call last):
+    TypeError: Argument 'a' has incorrect type (expected list, got int)
+    """
+    print(typeof(a), typeof(b), typeof(c), typeof(d))
+    a.append(b)
+    a.append(c)
+    a.append(d)
+    return a
+
+
+def pytypes_def(a: list, b: int = 2, c: long = 3, d: float = 4) -> list:
+    """
+    >>> pytypes_def([1])
+    ('list object', 'Python object', 'Python object', 'double')
+    [1, 2, 3, 4.0]
+    >>> pytypes_def([1], 3)
+    ('list object', 'Python object', 'Python object', 'double')
+    [1, 3, 3, 4.0]
     >>> pytypes_def(123)
     Traceback (most recent call last):
     TypeError: Argument 'a' has incorrect type (expected list, got int)
@@ -22,13 +43,13 @@ def pytypes_def(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type':
     return a
 
 
-cpdef pytypes_cpdef(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4):
+cpdef pytypes_cpdef(a: list, b: int = 2, c: long = 3, d: float = 4):
     """
     >>> pytypes_cpdef([1])
-    ('list object', 'int', 'long', 'float')
+    ('list object', 'Python object', 'Python object', 'double')
     [1, 2, 3, 4.0]
     >>> pytypes_cpdef([1], 3)
-    ('list object', 'int', 'long', 'float')
+    ('list object', 'Python object', 'Python object', 'double')
     [1, 3, 3, 4.0]
     >>> pytypes_cpdef(123)
     Traceback (most recent call last):
@@ -41,7 +62,7 @@ cpdef pytypes_cpdef(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'ty
     return a
 
 
-cdef c_pytypes_cdef(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4):
+cdef c_pytypes_cdef(a: list, b: int = 2, c: long = 3, d: float = 4):
     print(typeof(a), typeof(b), typeof(c), typeof(d))
     a.append(b)
     a.append(c)
@@ -52,10 +73,10 @@ cdef c_pytypes_cdef(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'ty
 def pytypes_cdef(a, b=2, c=3, d=4):
     """
     >>> pytypes_cdef([1])
-    ('list object', 'int', 'long', 'float')
+    ('list object', 'Python object', 'Python object', 'double')
     [1, 2, 3, 4.0]
     >>> pytypes_cdef([1], 3)
-    ('list object', 'int', 'long', 'float')
+    ('list object', 'Python object', 'Python object', 'double')
     [1, 3, 3, 4.0]
     >>> pytypes_cdef(123)   # doctest: +ELLIPSIS
     Traceback (most recent call last):
@@ -64,6 +85,25 @@ def pytypes_cdef(a, b=2, c=3, d=4):
     return c_pytypes_cdef(a, b, c, d)
 
 
+def ctypes_def(a: list, b: cython.int = 2, c: cython.long = 3, d: cython.float = 4) -> list:
+    """
+    >>> pytypes_def([1])
+    ('list object', 'Python object', 'Python object', 'double')
+    [1, 2, 3, 4.0]
+    >>> pytypes_def([1], 3)
+    ('list object', 'Python object', 'Python object', 'double')
+    [1, 3, 3, 4.0]
+    >>> pytypes_def(123)
+    Traceback (most recent call last):
+    TypeError: Argument 'a' has incorrect type (expected list, got int)
+    """
+    print(typeof(a), typeof(b), typeof(c), typeof(d))
+    a.append(b)
+    a.append(c)
+    a.append(d)
+    return a
+
+
 def return_tuple_for_carray() -> tuple:
     """
     >>> return_tuple_for_carray()
@@ -72,3 +112,15 @@ def return_tuple_for_carray() -> tuple:
     cdef int[3] x
     x = [1, 2, 3]
     return x
+
+
+_WARNINGS = """
+8:32: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.
+8:47: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.
+8:56: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.
+8:77: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.
+8:85: Python type declaration in signature annotation does not refer to a Python type
+8:85: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.
+# BUG:
+46:6: 'pytypes_cpdef' redeclared
+"""

tests/run/pep526_variable_annotations.py

-# mode: run
-# tag: pure3.6, warnings
 # cython: language_level=3
+# mode: run
+# tag: pure3.6, pep526, pep484, warnings
 
 import cython