Implement PEP 526: syntax for variable annotations.

Also parses variable annotations as type declarations.
Closes #1850.

Cython/Compiler/ExprNodes.py

@@ -254,9 +254,11 @@ class ExprNode(Node):
     #  result_is_used  boolean   indicates that the result will be dropped and the
     #  is_numpy_attribute   boolean   Is a Numpy module attribute
     #                            result_code/temp_result can safely be set to None
+    #  annotation   ExprNode or None    PEP526 annotation for names or expressions
 
     result_ctype = None
     type = None
+    annotation = None
     temp_code = None
     old_temp = None # error checker for multiple frees etc.
     use_managed_ref = True # can be set by optimisation transforms
@@ -1830,6 +1832,38 @@ class NameNode(AtomicExprNode):
 
         return super(NameNode, self).coerce_to(dst_type, env)
 
+    def declare_from_annotation(self, env, as_target=False):
+        """Implements PEP 526 annotation typing in a fairly relaxed way.
+
+        Annotations are ignored for global variables, Python class attributes and already declared variables.
+        String literals are allowed and ignored.
+        The ambiguous Python types 'int' and 'long' are ignored and the 'cython.int' form must be used instead.
+        """
+        if env.is_module_scope or env.is_py_class_scope:
+            # annotations never create global cdef names and Python classes don't support them anyway
+            return
+        name = self.name
+        if self.entry or env.lookup_here(name) is not None:
+            # already declared => ignore annotation
+            return
+
+        annotation = self.annotation
+        atype = annotation.analyse_as_type(env)
+        if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'):
+            # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations
+            if atype in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type):
+                atype = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type
+        elif annotation.is_string_literal:
+            # name: "description" => not a type, but still a declared variable or attribute
+            atype = None
+        elif atype is None:
+            # annotations always make variables local => ignore and leave to type inference
+            warning(annotation.pos, "Unknown type declaration in annotation, ignoring")
+
+        if atype is None:
+            atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type
+        self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target)
+
     def analyse_as_module(self, env):
         # Try to interpret this as a reference to a cimported module.
         # Returns the module scope, or None.
@@ -1869,6 +1903,9 @@ class NameNode(AtomicExprNode):
     def analyse_target_declaration(self, env):
         if not self.entry:
             self.entry = env.lookup_here(self.name)
+        if not self.entry and self.annotation is not None:
+            # name : type = ...
+            self.declare_from_annotation(env, as_target=True)
         if not self.entry:
             if env.directives['warn.undeclared']:
                 warning(self.pos, "implicit declaration of '%s'" % self.name, 1)

Cython/Compiler/Nodes.py

@@ -4772,12 +4772,13 @@ class ExprStatNode(StatNode):
 
     def analyse_declarations(self, env):
         from . import ExprNodes
-        if isinstance(self.expr, ExprNodes.GeneralCallNode):
-            func = self.expr.function.as_cython_attribute()
+        expr = self.expr
+        if isinstance(expr, ExprNodes.GeneralCallNode):
+            func = expr.function.as_cython_attribute()
             if func == u'declare':
-                args, kwds = self.expr.explicit_args_kwds()
+                args, kwds = expr.explicit_args_kwds()
                 if len(args):
-                    error(self.expr.pos, "Variable names must be specified.")
+                    error(expr.pos, "Variable names must be specified.")
                 for var, type_node in kwds.key_value_pairs:
                     type = type_node.analyse_as_type(env)
                     if type is None:
@@ -4785,6 +4786,14 @@ class ExprStatNode(StatNode):
                     else:
                         env.declare_var(var.value, type, var.pos, is_cdef=True)
                 self.__class__ = PassStatNode
+        elif expr.annotation is not None:
+            if expr.is_name:
+                # non-code variable annotation, e.g. "name: type"
+                expr.declare_from_annotation(env)
+                self.__class__ = PassStatNode
+            elif expr.is_attribute or expr.is_subscript:
+                # unused expression with annotation, e.g. "a[0]: type" or "a.xyz : type"
+                self.__class__ = PassStatNode
 
     def analyse_expressions(self, env):
         self.expr.result_is_used = False  # hint that .result() may safely be left empty

Cython/Compiler/Parsing.py

@@ -1485,13 +1485,17 @@ def p_nonlocal_statement(s):
 
 
 def p_expression_or_assignment(s):
-    expr_list = [p_testlist_star_expr(s)]
-    if s.sy == '=' and expr_list[0].is_starred:
+    expr = p_testlist_star_expr(s)
+    if s.sy == ':' and (expr.is_name or expr.is_subscript or expr.is_attribute):
+        s.next()
+        expr.annotation = p_test(s)
+    if s.sy == '=' and expr.is_starred:
         # This is a common enough error to make when learning Cython to let
         # it fail as early as possible and give a very clear error message.
         s.error("a starred assignment target must be in a list or tuple"
                 " - maybe you meant to use an index assignment: var[0] = ...",
-                pos=expr_list[0].pos)
+                pos=expr.pos)
+    expr_list = [expr]
     while s.sy == '=':
         s.next()
         if s.sy == 'yield':

runtests.py

@@ -346,6 +346,7 @@ VER_DEP_MODULES = {
                                              ]),
     (3,5): (operator.lt, lambda x: x in ['run.py35_pep492_interop',
                                          'run.mod__spec__',
+                                         'run.pep526_variable_annotations',  # typing module
                                          ]),
 }
 
@@ -578,9 +579,16 @@ class TestBuilder(object):
             for test in self.build_tests(test_class, path, workdir,
                                          module, mode == 'error', tags):
                 suite.addTest(test)
+
             if mode == 'run' and ext == '.py' and not self.cython_only and not filename.startswith('test_'):
                 # additionally test file in real Python
-                suite.addTest(PureDoctestTestCase(module, os.path.join(path, filename)))
+                min_py_ver = [
+                    (int(pyver.group(1)), int(pyver.group(2)))
+                    for pyver in map(re.compile(r'pure([0-9]+)[.]([0-9]+)').match, tags['tag'])
+                    if pyver
+                ]
+                if not min_py_ver or any(sys.version_info >= min_ver for min_ver in min_py_ver):
+                    suite.addTest(PureDoctestTestCase(module, os.path.join(path, filename)))
 
         return suite
 

tests/run/cython3.pyx

@@ -513,4 +513,6 @@ def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwar
     >>> print(annotation_syntax.__annotations__['return'])
     ret
     """
-    return a+b
+    result : int = a + b
+
+    return result

tests/run/pep526_variable_annotations.py

+# mode: run
+# tag: pure3.6, warnings
+# cython: language_level=3
+
+import cython
+
+from typing import Dict, List, TypeVar, Optional, Generic, Tuple
+try:
+    from typing import ClassVar
+except ImportError:
+    ClassVar = Optional  # fake it in Py3.5
+
+
+var = 1  # type: annotation
+var: int = 2
+fvar: float = 1.2
+some_number: cython.int    # variable without initial value
+some_list: List[int] = []  # variable with initial value
+t: Tuple[int, ...] = (1, 2, 3)
+body: Optional[List[str]]
+descr_only : "descriptions are allowed but ignored"
+
+
+some_number = 5
+body = None
+
+
+def f():
+    """
+    >>> f()
+    (2, 1.5, [], (1, 2, 3))
+    """
+    var = 1  # type: annotation
+    var: int = 2
+    fvar: float = 1.5
+    some_number: cython.int    # variable without initial value
+    some_list: List[int] = []  # variable with initial value
+    t: Tuple[int, ...] = (1, 2, 3)
+    body: Optional[List[str]]
+    descr_only: "descriptions are allowed but ignored"
+
+    return var, fvar, some_list, t
+
+
+class BasicStarship(object):
+    """
+    >>> bs = BasicStarship(5)
+    >>> bs.damage
+    5
+    >>> bs.captain
+    'Picard'
+    >>> bs.stats
+    {}
+    """
+    captain: str = 'Picard'               # instance variable with default
+    damage: cython.int                    # instance variable without default
+    stats: ClassVar[Dict[str, int]] = {}  # class variable
+    descr_only: "descriptions are allowed but ignored"
+
+    def __init__(self, damage):
+        self.damage = damage
+
+
+@cython.cclass
+class BasicStarshipExt(object):
+    """
+    >>> bs = BasicStarshipExt(5)
+    >>> bs.test()
+    (5, 'Picard', {})
+    """
+    captain: str = 'Picard'               # instance variable with default
+    damage: cython.int                    # instance variable without default
+    stats: ClassVar[Dict[str, int]] = {}  # class variable
+    descr_only: "descriptions are allowed but ignored"
+
+    def __init__(self, damage):
+        self.damage = damage
+
+    def test(self):
+        return self.damage, self.captain, self.stats
+
+
+T = TypeVar('T')
+
+
+class Box(Generic[T]):
+    def __init__(self, content):
+        self.content: T = content
+
+
+box = Box(content=5)
+
+
+class Cls(object):
+    pass
+
+
+c = Cls()
+c.x: int = 0  # Annotates c.x with int.
+c.y: int      # Annotates c.y with int.
+
+d = {}
+d['a']: int = 0  # Annotates d['a'] with int.
+d['b']: int      # Annotates d['b'] with int.
+
+(x): int      # Annotates x with int, (x) treated as expression by compiler.
+(y): int = 0  # Same situation here.
+
+
+
+_WARNINGS = """
+37:19: Unknown type declaration in annotation, ignoring
+38:12: Unknown type declaration in annotation, ignoring
+39:18: Unknown type declaration in annotation, ignoring
+73:19: Unknown type declaration in annotation, ignoring
+"""