Merge pull request #1781 from cython/dalcinl-embedsignature

Update embedsignature directive to emit function annotations

Cython/CodeWriter.py

@@ -519,3 +519,298 @@ class PxdWriter(DeclarationWriter):
 
     def visit_StatNode(self, node):
         pass
+
+
+class ExpressionWriter(TreeVisitor):
+
+    def __init__(self, result=None):
+        super(ExpressionWriter, self).__init__()
+        if result is None:
+            result = u""
+        self.result = result
+        self.precedence = [0]
+
+    def write(self, tree):
+        self.visit(tree)
+        return self.result
+
+    def put(self, s):
+        self.result += s
+
+    def remove(self, s):
+        if self.result.endswith(s):
+            self.result = self.result[:-len(s)]
+
+    def comma_separated_list(self, items):
+        if len(items) > 0:
+            for item in items[:-1]:
+                self.visit(item)
+                self.put(u", ")
+            self.visit(items[-1])
+
+    def visit_Node(self, node):
+        raise AssertionError("Node not handled by serializer: %r" % node)
+
+    def visit_NameNode(self, node):
+        self.put(node.name)
+
+    def visit_NoneNode(self, node):
+        self.put(u"None")
+
+    def visit_EllipsisNode(self, node):
+        self.put(u"...")
+
+    def visit_BoolNode(self, node):
+        self.put(str(node.value))
+
+    def visit_ConstNode(self, node):
+        self.put(str(node.value))
+
+    def visit_ImagNode(self, node):
+        self.put(node.value)
+        self.put(u"j")
+
+    def emit_string(self, node, prefix=u""):
+        repr_val = repr(node.value)
+        if repr_val[0] in 'ub':
+            repr_val = repr_val[1:]
+        self.put(u"%s%s" % (prefix, repr_val))
+
+    def visit_BytesNode(self, node):
+        self.emit_string(node, u"b")
+
+    def visit_StringNode(self, node):
+        self.emit_string(node)
+
+    def visit_UnicodeNode(self, node):
+        self.emit_string(node, u"u")
+
+    def emit_sequence(self, node, parens=(u"", u"")):
+        open_paren, close_paren = parens
+        items = node.subexpr_nodes()
+        self.put(open_paren)
+        self.comma_separated_list(items)
+        self.put(close_paren)
+
+    def visit_ListNode(self, node):
+        self.emit_sequence(node, u"[]")
+
+    def visit_TupleNode(self, node):
+        self.emit_sequence(node, u"()")
+
+    def visit_SetNode(self, node):
+        if len(node.subexpr_nodes()) > 0:
+            self.emit_sequence(node, u"{}")
+        else:
+            self.put(u"set()")
+
+    def visit_DictNode(self, node):
+        self.emit_sequence(node, u"{}")
+
+    def visit_DictItemNode(self, node):
+        self.visit(node.key)
+        self.put(u": ")
+        self.visit(node.value)
+
+    unop_precedence = {
+        'not': 3, '!': 3,
+        '+': 11, '-': 11, '~': 11,
+    }
+    binop_precedence = {
+        'or': 1,
+        'and': 2,
+        # unary: 'not': 3, '!': 3,
+        'in': 4, 'not_in': 4, 'is': 4, 'is_not': 4, '<': 4, '<=': 4, '>': 4, '>=': 4, '!=': 4, '==': 4,
+        '|': 5,
+        '^': 6,
+        '&': 7,
+        '<<': 8, '>>': 8,
+        '+': 9, '-': 9,
+        '*': 10, '@': 10, '/': 10, '//': 10, '%': 10,
+        # unary: '+': 11, '-': 11, '~': 11
+        '**': 12,
+    }
+
+    def operator_enter(self, new_prec):
+        old_prec = self.precedence[-1]
+        if old_prec > new_prec:
+            self.put(u"(")
+        self.precedence.append(new_prec)
+
+    def operator_exit(self):
+        old_prec, new_prec = self.precedence[-2:]
+        if old_prec > new_prec:
+            self.put(u")")
+        self.precedence.pop()
+
+    def visit_NotNode(self, node):
+        op = 'not'
+        prec = self.unop_precedence[op]
+        self.operator_enter(prec)
+        self.put(u"not ")
+        self.visit(node.operand)
+        self.operator_exit()
+
+    def visit_UnopNode(self, node):
+        op = node.operator
+        prec = self.unop_precedence[op]
+        self.operator_enter(prec)
+        self.put(u"%s" % node.operator)
+        self.visit(node.operand)
+        self.operator_exit()
+
+    def visit_BinopNode(self, node):
+        op = node.operator
+        prec = self.binop_precedence.get(op, 0)
+        self.operator_enter(prec)
+        self.visit(node.operand1)
+        self.put(u" %s " % op.replace('_', ' '))
+        self.visit(node.operand2)
+        self.operator_exit()
+
+    def visit_BoolBinopNode(self, node):
+        self.visit_BinopNode(node)
+
+    def visit_PrimaryCmpNode(self, node):
+        self.visit_BinopNode(node)
+
+    def visit_IndexNode(self, node):
+        self.visit(node.base)
+        self.put(u"[")
+        if isinstance(node.index, TupleNode):
+            self.emit_sequence(node.index)
+        else:
+            self.visit(node.index)
+        self.put(u"]")
+
+    def visit_SliceIndexNode(self, node):
+        self.visit(node.base)
+        self.put(u"[")
+        if node.start:
+            self.visit(node.start)
+        self.put(u":")
+        if node.stop:
+            self.visit(node.stop)
+        if node.slice:
+            self.put(u":")
+            self.visit(node.slice)
+        self.put(u"]")
+
+    def visit_SliceNode(self, node):
+        if not node.start.is_none:
+            self.visit(node.start)
+        self.put(u":")
+        if not node.stop.is_none:
+            self.visit(node.stop)
+        if not node.step.is_none:
+            self.put(u":")
+            self.visit(node.step)
+
+    def visit_CondExprNode(self, node):
+        self.visit(node.true_val)
+        self.put(u" if ")
+        self.visit(node.test)
+        self.put(u" else ")
+        self.visit(node.false_val)
+
+    def visit_AttributeNode(self, node):
+        self.visit(node.obj)
+        self.put(u".%s" % node.attribute)
+
+    def visit_SimpleCallNode(self, node):
+        self.visit(node.function)
+        self.put(u"(")
+        self.comma_separated_list(node.args)
+        self.put(")")
+
+    def emit_pos_args(self, node):
+        if node is None:
+            return
+        if isinstance(node, AddNode):
+            self.emit_pos_args(node.operand1)
+            self.emit_pos_args(node.operand2)
+        elif isinstance(node, TupleNode):
+            for expr in node.subexpr_nodes():
+                self.visit(expr)
+                self.put(u", ")
+        elif isinstance(node, AsTupleNode):
+            self.put("*")
+            self.visit(node.arg)
+            self.put(u", ")
+        else:
+            self.visit(node)
+            self.put(u", ")
+
+    def emit_kwd_args(self, node):
+        if node is None:
+            return
+        if isinstance(node, MergedDictNode):
+            for expr in node.subexpr_nodes():
+                self.emit_kwd_args(expr)
+        elif isinstance(node, DictNode):
+            for expr in node.subexpr_nodes():
+                self.put(u"%s=" % expr.key.value)
+                self.visit(expr.value)
+                self.put(u", ")
+        else:
+            self.put(u"**")
+            self.visit(node)
+            self.put(u", ")
+
+    def visit_GeneralCallNode(self, node):
+        self.visit(node.function)
+        self.put(u"(")
+        self.emit_pos_args(node.positional_args)
+        self.emit_kwd_args(node.keyword_args)
+        self.remove(u", ")
+        self.put(")")
+
+    def emit_comprehension(self, body, target,
+                           sequence, condition,
+                           parens=(u"", u"")):
+        open_paren, close_paren = parens
+        self.put(open_paren)
+        self.visit(body)
+        self.put(u" for ")
+        self.visit(target)
+        self.put(u" in ")
+        self.visit(sequence)
+        if condition:
+            self.put(u" if ")
+            self.visit(condition)
+        self.put(close_paren)
+
+    def visit_ComprehensionAppendNode(self, node):
+        self.visit(node.expr)
+
+    def visit_DictComprehensionAppendNode(self, node):
+        self.visit(node.key_expr)
+        self.put(u": ")
+        self.visit(node.value_expr)
+
+    def visit_ComprehensionNode(self, node):
+        tpmap = {'list': u"[]", 'dict': u"{}", 'set': u"{}"}
+        parens = tpmap[node.type.py_type_name()]
+        body = node.loop.body
+        target = node.loop.target
+        sequence = node.loop.iterator.sequence
+        condition = None
+        if hasattr(body, 'if_clauses'):
+            # type(body) is Nodes.IfStatNode
+            condition = body.if_clauses[0].condition
+            body = body.if_clauses[0].body
+        self.emit_comprehension(body, target, sequence, condition, parens)
+
+    def visit_GeneratorExpressionNode(self, node):
+        body = node.loop.body
+        target = node.loop.target
+        sequence = node.loop.iterator.sequence
+        condition = None
+        if hasattr(body, 'if_clauses'):
+            # type(body) is Nodes.IfStatNode
+            condition = body.if_clauses[0].condition
+            body = body.if_clauses[0].body.expr.arg
+        elif hasattr(body, 'expr'):
+            # type(body) is Nodes.ExprStatNode
+            body = body.expr.arg
+        self.emit_comprehension(body, target, sequence, condition, u"()")

Cython/Compiler/AutoDocTransforms.py

-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 
 from .Visitor import CythonTransform
 from .StringEncoding import EncodedString
 from . import Options
 from . import PyrexTypes, ExprNodes
+from ..CodeWriter import ExpressionWriter
+
+
+class AnnotationWriter(ExpressionWriter):
+
+    def visit_Node(self, node):
+        self.put(u"<???>")
+
+    def visit_LambdaNode(self, node):
+        # XXX Should we do better?
+        self.put("<lambda>")
+
 
 class EmbedSignature(CythonTransform):
 
     def __init__(self, context):
         super(EmbedSignature, self).__init__(context)
-        self.denv = None # XXX
         self.class_name = None
         self.class_node = None
 
-    unop_precedence = 11
-    binop_precedence = {
-        'or': 1,
-        'and': 2,
-        'not': 3,
-        'in': 4, 'not in': 4, 'is': 4, 'is not': 4, '<': 4, '<=': 4, '>': 4, '>=': 4, '!=': 4, '==': 4,
-        '|': 5,
-        '^': 6,
-        '&': 7,
-        '<<': 8, '>>': 8,
-        '+': 9, '-': 9,
-        '*': 10, '/': 10, '//': 10, '%': 10,
-        # unary: '+': 11, '-': 11, '~': 11
-        '**': 12}
-
-    def _fmt_expr_node(self, node, precedence=0):
-        if isinstance(node, ExprNodes.BinopNode) and not node.inplace:
-            new_prec = self.binop_precedence.get(node.operator, 0)
-            result = '%s %s %s' % (self._fmt_expr_node(node.operand1, new_prec),
-                                   node.operator,
-                                   self._fmt_expr_node(node.operand2, new_prec))
-            if precedence > new_prec:
-                result = '(%s)' % result
-        elif isinstance(node, ExprNodes.UnopNode):
-            result = '%s%s' % (node.operator,
-                               self._fmt_expr_node(node.operand, self.unop_precedence))
-            if precedence > self.unop_precedence:
-                result = '(%s)' % result
-        elif isinstance(node, ExprNodes.AttributeNode):
-            result = '%s.%s' % (self._fmt_expr_node(node.obj), node.attribute)
-        else:
-            result = node.name
+    def _fmt_expr(self, node):
+        writer = AnnotationWriter()
+        result = writer.write(node)
+        # print(type(node).__name__, '-->', result)
         return result
 
-    def _fmt_arg_defv(self, arg):
-        default_val = arg.default
-        if not default_val:
-            return None
-        if isinstance(default_val, ExprNodes.NullNode):
-            return 'NULL'
-        try:
-            denv = self.denv  # XXX
-            ctval = default_val.compile_time_value(self.denv)
-            repr_val = repr(ctval)
-            if isinstance(default_val, ExprNodes.UnicodeNode):
-                if repr_val[:1] != 'u':
-                    return u'u%s' % repr_val
-            elif isinstance(default_val, ExprNodes.BytesNode):
-                if repr_val[:1] != 'b':
-                    return u'b%s' % repr_val
-            elif isinstance(default_val, ExprNodes.StringNode):
-                if repr_val[:1] in 'ub':
-                    return repr_val[1:]
-            return repr_val
-        except Exception:
-            try:
-                return self._fmt_expr_node(default_val)
-            except AttributeError:
-                return '<???>'
-
     def _fmt_arg(self, arg):
         if arg.type is PyrexTypes.py_object_type or arg.is_self_arg:
             doc = arg.name
         else:
             doc = arg.type.declaration_code(arg.name, for_display=1)
-        if arg.default:
-            arg_defv = self._fmt_arg_defv(arg)
-            if arg_defv:
-                doc = doc + ('=%s' % arg_defv)
+
+        if arg.annotation:
+            annotation = self._fmt_expr(arg.annotation)
+            doc = doc + (': %s' % annotation)
+            if arg.default:
+                default = self._fmt_expr(arg.default)
+                doc = doc + (' = %s' % default)
+        elif arg.default:
+            default = self._fmt_expr(arg.default)
+            doc = doc + ('=%s' % default)
         return doc
 
+    def _fmt_star_arg(self, arg):
+        arg_doc = arg.name
+        if arg.annotation:
+            annotation = self._fmt_expr(arg.annotation)
+            arg_doc = arg_doc + (': %s' % annotation)
+        return arg_doc
+
     def _fmt_arglist(self, args,
                      npargs=0, pargs=None,
                      nkargs=0, kargs=None,
@@ -94,11 +64,13 @@ class EmbedSignature(CythonTransform):
                 arg_doc = self._fmt_arg(arg)
                 arglist.append(arg_doc)
         if pargs:
-            arglist.insert(npargs, '*%s' % pargs.name)
+            arg_doc = self._fmt_star_arg(pargs)
+            arglist.insert(npargs, '*%s' % arg_doc)
         elif nkargs:
             arglist.insert(npargs, '*')
         if kargs:
-            arglist.append('**%s' % kargs.name)
+            arg_doc = self._fmt_star_arg(kargs)
+            arglist.append('**%s' % arg_doc)
         return arglist
 
     def _fmt_ret_type(self, ret):
@@ -110,6 +82,7 @@ class EmbedSignature(CythonTransform):
     def _fmt_signature(self, cls_name, func_name, args,
                        npargs=0, pargs=None,
                        nkargs=0, kargs=None,
+                       return_expr=None,
                        return_type=None, hide_self=False):
         arglist = self._fmt_arglist(args,
                                     npargs, pargs,
@@ -119,10 +92,13 @@ class EmbedSignature(CythonTransform):
         func_doc = '%s(%s)' % (func_name, arglist_doc)
         if cls_name:
             func_doc = '%s.%s' % (cls_name, func_doc)
-        if return_type:
+        ret_doc = None
+        if return_expr:
+            ret_doc = self._fmt_expr(return_expr)
+        elif return_type:
             ret_doc = self._fmt_ret_type(return_type)
-            if ret_doc:
-                func_doc = '%s -> %s' % (func_doc, ret_doc)
+        if ret_doc:
+            func_doc = '%s -> %s' % (func_doc, ret_doc)
         return func_doc
 
     def _embed_signature(self, signature, node_doc):
@@ -177,6 +153,7 @@ class EmbedSignature(CythonTransform):
             class_name, func_name, node.args,
             npargs, node.star_arg,
             nkargs, node.starstar_arg,
+            return_expr=node.return_type_annotation,
             return_type=None, hide_self=hide_self)
         if signature:
             if is_constructor:

tests/run/embedsignatures.pyx

@@ -80,6 +80,9 @@ __doc__ = ur"""
     >>> print (Ext.m.__doc__)
     Ext.m(self, a=u'spam')
 
+    >>> print (Ext.n.__doc__)
+    Ext.n(self, a: int, b: float = 1.0, *args: tuple, **kwargs: dict) -> (None, True)
+
     >>> print (Ext.get_int.__doc__)
     Ext.get_int(self) -> int
 
@@ -185,7 +188,7 @@ __doc__ = ur"""
     f_defexpr4(int x=(Ext.CONST1 + FLAG1) * Ext.CONST2)
 
     >>> print(funcdoc(f_defexpr5))
-    f_defexpr5(int x=4)
+    f_defexpr5(int x=2 + 2)
 
     >>> print(funcdoc(f_charptr_null))
     f_charptr_null(char *s=NULL) -> char *
@@ -259,6 +262,9 @@ cdef class Ext:
     def m(self, a=u'spam'):
         pass
 
+    def n(self, a: int, b: float = 1.0, *args: tuple, **kwargs: dict) -> (None, True):
+        pass
+
     cpdef int get_int(self):
         return 0
 
@@ -388,3 +394,132 @@ cpdef (char*) f_charptr_null(char* s=NULL):
 # no signatures for lambda functions
 lambda_foo = lambda x: 10
 lambda_bar = lambda x: 20
+
+
+cdef class Foo:
+    def m00(self, a: None) ->  None: pass
+    def m01(self, a: ...) ->  Ellipsis: pass
+    def m02(self, a: True, b: False) ->  bool: pass
+    def m03(self, a: 42, b: +42, c: -42) ->  int : pass  # XXX +42 -> 42
+    def m04(self, a: 3.14, b: +3.14, c: -3.14) -> float : pass
+    def m05(self, a: 1 + 2j, b: +2j, c: -2j) -> complex : pass
+    def m06(self, a: "abc", b: b"abc", c: u"abc") -> (str, bytes, unicode) : pass
+    def m07(self, a: [1, 2, 3], b: []) -> list: pass
+    def m08(self, a: (1, 2, 3), b: ()) -> tuple: pass
+    def m09(self, a: {1, 2, 3}, b: {i for i in ()}) -> set: pass
+    def m10(self, a: {1: 1, 2: 2, 3: 3}, b: {}) -> dict: pass
+   #def m11(self, a: [str(i) for i in range(3)]): pass  # Issue 1782
+    def m12(self, a: (str(i) for i in range(3))): pass
+    def m13(self, a: (str(i) for i in range(3) if bool(i))): pass
+    def m14(self, a: {str(i) for i in range(3)}): pass
+    def m15(self, a: {str(i) for i in range(3) if bool(i)}): pass
+    def m16(self, a: {str(i): id(i) for i in range(3)}): pass
+    def m17(self, a: {str(i): id(i) for i in range(3) if bool(i)}): pass
+    def m18(self, a: dict.update(x=42, **dict(), **{})): pass
+    def m19(self, a: sys is None, b: sys is not None): pass
+    def m20(self, a: sys in [], b: sys not in []): pass
+    def m21(self, a: (sys or sys) and sys, b: not (sys or sys)): pass
+    def m22(self, a: 42 if sys else None): pass
+    def m23(self, a: +int(), b: -int(), c: ~int()): pass
+    def m24(self, a: (1+int(2))*3+(4*int(5))**(1+0.0/1)): pass
+    def m25(self, a: list(range(3))[:]): pass
+    def m26(self, a: list(range(3))[1:]): pass
+    def m27(self, a: list(range(3))[:1]): pass
+    def m28(self, a: list(range(3))[::1]): pass
+    def m29(self, a: list(range(3))[0:1:1]): pass
+    def m30(self, a: list(range(3))[7, 3:2:1, ...]): pass
+
+__doc__ += ur"""
+>>> print(Foo.m00.__doc__)
+Foo.m00(self, a: None) -> None
+
+>>> print(Foo.m01.__doc__)
+Foo.m01(self, a: ...) -> Ellipsis
+
+>>> print(Foo.m02.__doc__)
+Foo.m02(self, a: True, b: False) -> bool
+
+>>> print(Foo.m03.__doc__)
+Foo.m03(self, a: 42, b: 42, c: -42) -> int
+
+>>> print(Foo.m04.__doc__)
+Foo.m04(self, a: 3.14, b: 3.14, c: -3.14) -> float
+
+>>> print(Foo.m05.__doc__)
+Foo.m05(self, a: 1 + 2j, b: +2j, c: -2j) -> complex
+
+>>> print(Foo.m06.__doc__)
+Foo.m06(self, a: 'abc', b: b'abc', c: u'abc') -> (str, bytes, unicode)
+
+>>> print(Foo.m07.__doc__)
+Foo.m07(self, a: [1, 2, 3], b: []) -> list
+
+>>> print(Foo.m08.__doc__)
+Foo.m08(self, a: (1, 2, 3), b: ()) -> tuple
+
+>>> print(Foo.m09.__doc__)
+Foo.m09(self, a: {1, 2, 3}, b: set()) -> set
+
+>>> print(Foo.m10.__doc__)
+Foo.m10(self, a: {1: 1, 2: 2, 3: 3}, b: {}) -> dict
+
+# >>> print(Foo.m11.__doc__)
+# Foo.m11(self, a: [str(i) for i in range(3)])
+
+>>> print(Foo.m12.__doc__)
+Foo.m12(self, a: (str(i) for i in range(3)))
+
+>>> print(Foo.m13.__doc__)
+Foo.m13(self, a: (str(i) for i in range(3) if bool(i)))
+
+>>> print(Foo.m14.__doc__)
+Foo.m14(self, a: {str(i) for i in range(3)})
+
+>>> print(Foo.m15.__doc__)
+Foo.m15(self, a: {str(i) for i in range(3) if bool(i)})
+
+>>> print(Foo.m16.__doc__)
+Foo.m16(self, a: {str(i): id(i) for i in range(3)})
+
+>>> print(Foo.m17.__doc__)
+Foo.m17(self, a: {str(i): id(i) for i in range(3) if bool(i)})
+
+>>> print(Foo.m18.__doc__)
+Foo.m18(self, a: dict.update(x=42, **dict()))
+
+>>> print(Foo.m19.__doc__)
+Foo.m19(self, a: sys is None, b: sys is not None)
+
+>>> print(Foo.m20.__doc__)
+Foo.m20(self, a: sys in [], b: sys not in [])
+
+>>> print(Foo.m21.__doc__)
+Foo.m21(self, a: (sys or sys) and sys, b: not (sys or sys))
+
+>>> print(Foo.m22.__doc__)
+Foo.m22(self, a: 42 if sys else None)
+
+>>> print(Foo.m23.__doc__)
+Foo.m23(self, a: +int(), b: -int(), c: ~int())
+
+>>> print(Foo.m24.__doc__)
+Foo.m24(self, a: (1 + int(2)) * 3 + (4 * int(5)) ** (1 + 0.0 / 1))
+
+>>> print(Foo.m25.__doc__)
+Foo.m25(self, a: list(range(3))[:])
+
+>>> print(Foo.m26.__doc__)
+Foo.m26(self, a: list(range(3))[1:])
+
+>>> print(Foo.m27.__doc__)
+Foo.m27(self, a: list(range(3))[:1])
+
+>>> print(Foo.m28.__doc__)
+Foo.m28(self, a: list(range(3))[::1])
+
+>>> print(Foo.m29.__doc__)
+Foo.m29(self, a: list(range(3))[0:1:1])
+
+>>> print(Foo.m30.__doc__)
+Foo.m30(self, a: list(range(3))[7, 3:2:1, ...])
+"""