Automatic embedding of signatures in docstring (#2)

Cython/Compiler/AutoDocTransforms.py

+import re
+
+from Cython.Compiler.Visitor import CythonTransform
+from Cython.Compiler.Nodes import DefNode, CFuncDefNode
+from Cython.Compiler.Errors import CompileError
+from Cython.Compiler.StringEncoding import EncodedString
+from Cython.Compiler import Options
+
+
+class EmbedSignature(CythonTransform):
+
+    SPECIAL_METHOD_RE = re.compile(r'__\w+__')
+
+    def __init__(self, context):
+        super(EmbedSignature, self).__init__(context)
+        self.denv = None # XXX
+        self.is_in_class = False
+        self.class_name = None
+
+    def _fmt_arg_type(self, arg):
+        try:
+            return arg.base_type.name
+        except AttributeError:
+            return ""
+
+    def _fmt_arg_name(self, arg):
+        try:
+            return arg.declarator.name
+        except AttributeError:
+            return arg.declarator.base.name
+
+    def _fmt_arg_defv(self, arg):
+        if not arg.default:
+            return None
+        try:
+            denv = self.denv  # XXX
+            ctval = arg.default.compile_time_value(self.denv)
+            return '%s' % ctval
+        except Exception:
+            try:
+                return arg.default.name # XXX
+            except AttributeError:
+                return '<???>'
+
+    def _fmt_arg(self, arg):
+        arg_type = self._fmt_arg_type(arg)
+        arg_name = self._fmt_arg_name(arg)
+        arg_defv = self._fmt_arg_defv(arg)
+        doc = arg_name
+        if arg_type:
+            doc = ('%s ' % arg_type) + doc
+        if arg_defv:
+            doc = doc + ('=%s' % arg_defv)
+        return doc
+
+    def _fmt_arglist(self, args,
+                     npargs=0, pargs=None,
+                     nkargs=0, kargs=None):
+        arglist = []
+        for arg in args:
+            arg_doc = self._fmt_arg(arg)
+            arglist.append(arg_doc)
+        if pargs:
+            arglist.insert(npargs, '*%s' % pargs.name)
+        elif nkargs:
+            arglist.insert(npargs, '*')
+        if kargs:
+            arglist.append('**%s' % kargs.name)
+        return arglist
+
+    def _fmt_signature(self, cls_name, func_name, args,
+                       npargs=0, pargs=None,
+                       nkargs=0, kargs=None,
+                       return_type=None):
+        arglist = self._fmt_arglist(args,
+                                    npargs, pargs,
+                                    nkargs, kargs)
+        arglist = ', '.join(arglist)
+        func_doc = '%s(%s)' % (func_name, arglist)
+        if cls_name:
+            func_doc = ('%s.' % cls_name) + func_doc
+        if return_type:
+            func_doc = func_doc + ' -> %s' % return_type
+        return func_doc
+
+    def _embed_signature(self, signature, node_doc):
+        if node_doc:
+            return signature + '\n' + node_doc
+        else:
+            return signature
+
+    def visit_ClassDefNode(self, node):
+        oldincls = self.is_in_class
+        oldname = self.class_name
+        self.is_in_class = True
+        try:
+            # PyClassDefNode
+            self.class_name = node.name
+        except AttributeError:
+            # CClassDefNode
+            self.class_name = node.class_name
+        self.visitchildren(node)
+        self.is_in_class = oldincls
+        self.class_name = oldname
+        return node
+
+    def visit_FuncDefNode(self, node):
+        signature = None
+        if type(node) is DefNode: # def FOO(...):
+            special_method = (self.is_in_class and \
+                              self.SPECIAL_METHOD_RE.match(node.name))
+            if not special_method:
+                nkargs = getattr(node, 'num_kwonly_args', 0)
+                npargs = len(node.args) - nkargs
+                signature = self._fmt_signature(
+                    self.class_name, node.name, node.args,
+                    npargs, node.star_arg,
+                    nkargs, node.starstar_arg,
+                    return_type=None)
+        elif type(node) is CFuncDefNode:
+            if node.overridable: # cpdef FOO(...):
+                signature = self._fmt_signature(
+                    self.class_name, node.declarator.base.name,
+                    node.declarator.args,
+                    return_type=node.base_type.name)
+        else: # should not fall here ...
+            assert False
+        if signature:
+            if Options.docstrings and node.options['embedsignature']:
+                new_doc  = self._embed_signature(signature, node.doc)
+                node.doc = EncodedString(new_doc) # XXX
+        return node

Cython/Compiler/Main.py

@@ -80,6 +80,7 @@ class Context:
         from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
         from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
         from ParseTreeTransforms import ResolveOptions
+        from AutoDocTransforms import EmbedSignature
         from Optimize import FlattenInListTransform, SwitchTransform, OptimizeRefcounting
         from Buffer import IntroduceBufferAuxiliaryVars
         from ModuleNode import check_c_classes
@@ -96,6 +97,7 @@ class Context:
             PostParse(self),
             _specific_post_parse,
             ResolveOptions(self, self.pragma_overrides),
+            EmbedSignature(self),
             FlattenInListTransform(),
             WithTransform(self),
             DecoratorTransform(self),

Cython/Compiler/Options.py

@@ -56,11 +56,13 @@ c_line_in_traceback = 1
 
 # Declare pragmas
 option_types = {
-    'boundscheck' : bool
+    'boundscheck' : bool,
+    'embedsignature' : bool,
 }
 
 option_defaults = {
-    'boundscheck' : True
+    'boundscheck' : True,
+    'embedsignature' : False,
 }
 
 def parse_option_value(name, value):

Tools/cython-epydoc.py

+#! /usr/bin/env python
+
+# --------------------------------------------------------------------
+
+import re
+from epydoc import docstringparser as dsp
+
+CYTHON_SIGNATURE_RE = re.compile(
+    # Class name (for builtin methods)
+    r'^\s*((?P<class>\w+)\.)?' +
+    # The function name
+    r'(?P<func>\w+)' +
+    # The parameters
+    r'\(((?P<self>(?:self|cls|mcs)),?)?(?P<params>.*)\)' +
+    # The return value (optional)
+    r'(\s*(->)\s*(?P<return>\w+(?:\s*\w+)))?' +
+    # The end marker
+    r'\s*(?:\n|$)')
+
+parse_signature = dsp.parse_function_signature
+
+def parse_function_signature(func_doc, doc_source,
+                             docformat, parse_errors):
+    PYTHON_SIGNATURE_RE = dsp._SIGNATURE_RE
+    assert PYTHON_SIGNATURE_RE is not CYTHON_SIGNATURE_RE
+    try:
+        dsp._SIGNATURE_RE = CYTHON_SIGNATURE_RE
+        found = parse_signature(func_doc, doc_source,
+                                docformat, parse_errors)
+        dsp._SIGNATURE_RE = PYTHON_SIGNATURE_RE
+        if not found:
+            found = parse_signature(func_doc, doc_source,
+                                    docformat, parse_errors)
+        return found
+    finally:
+        dsp._SIGNATURE_RE = PYTHON_SIGNATURE_RE
+
+dsp.parse_function_signature = parse_function_signature
+
+# --------------------------------------------------------------------
+
+from epydoc.cli import cli
+cli()
+
+# --------------------------------------------------------------------

tests/run/embedsignatures.pyx

+#cython: embedsignature=True
+
+# note the r, we use \n below
+__doc__ = ur"""
+    >>> print (Ext.a.__doc__)
+    Ext.a(self)
+
+    >>> print (Ext.b.__doc__)
+    Ext.b(self, a, b, c)
+
+    >>> print (Ext.c.__doc__)
+    Ext.c(self, a, b, c=1)
+
+    >>> print (Ext.d.__doc__)
+    Ext.d(self, a, b, *, c=88)
+
+    >>> print (Ext.e.__doc__)
+    Ext.e(self, a, b, c=88, **kwds)
+
+    >>> print (Ext.f.__doc__)
+    Ext.f(self, a, b, *, c, d=42)
+
+    >>> print (Ext.g.__doc__)
+    Ext.g(self, a, b, *, c, d=42, e=17, f, **kwds)
+
+    >>> print (Ext.h.__doc__)
+    Ext.h(self, a, b, *args, c, d=42, e=17, f, **kwds)
+
+    >>> print (Ext.k.__doc__)
+    Ext.k(self, a, b, c=1, *args, d=42, e=17, f, **kwds)
+
+    >>> print (Ext.get_int.__doc__)
+    Ext.get_int(self) -> int
+
+    >>> print (Ext.get_float.__doc__)
+    Ext.get_float(self) -> float
+
+    >>> print (Ext.clone.__doc__)
+    Ext.clone(self) -> Ext
+
+    >>> print (foo.__doc__)
+    foo()
+
+    >>> with_doc_1.__doc__
+    'with_doc_1(a, b, c)\nExisting string'
+
+    >>> with_doc_2.__doc__
+    'with_doc_2(a, b, c)\n\n    Existing string\n    '
+
+    >>> types.__doc__
+    'types(Ext a, int b, int c, float d, e)'
+
+"""
+
+cdef class Ext:
+
+    def a(self):
+        pass
+
+    def b(self, a, b, c):
+        pass
+
+    def c(self, a, b, c=1):
+        pass
+
+    def d(self, a, b, *, c = 88):
+        pass
+
+    def e(self, a, b, c = 88, **kwds):
+        pass
+
+    def f(self, a, b, *, c, d = 42):
+        pass
+
+    def g(self, a, b, *, c, d = 42, e = 17, f, **kwds):
+        pass
+
+    def h(self, a, b, *args, c, d = 42, e = 17, f, **kwds):
+        pass
+
+    def k(self, a, b, c=1, *args, d = 42, e = 17, f, **kwds):
+        pass
+
+    cpdef int get_int(self):
+        return 0
+
+    cpdef float get_float(self):
+        return 0.0
+
+    cpdef Ext clone(self):
+        return Ext()
+
+def foo():
+    pass
+
+def types(Ext a, int b, unsigned short c, float d, e):
+    pass
+
+def with_doc_1(a, b, c):
+    """Existing string"""
+    pass
+
+def with_doc_2(a, b, c):
+    """
+    Existing string
+    """
+    pass