properly handle expressions at the beginning of func/class/etc. blocks that...

properly handle expressions at the beginning of func/class/etc. blocks that start with a string literal but are not docstrings

Cython/Compiler/ParseTreeTransforms.py

@@ -121,13 +121,20 @@ class NormalizeTree(CythonTransform):
     def visit_CStructOrUnionDefNode(self, node):
         return self.visit_StatNode(node, True)
 
-    # Eliminate PassStatNode
     def visit_PassStatNode(self, node):
+        """Eliminate PassStatNode"""
         if not self.is_in_statlist:
             return Nodes.StatListNode(pos=node.pos, stats=[])
         else:
             return []
 
+    def visit_ExprStatNode(self, node):
+        """Eliminate useless string literals"""
+        if node.expr.is_string_literal:
+            return None
+        self.visitchildren(node)
+        return node
+
     def visit_CDeclaratorNode(self, node):
         return node
 

Cython/Compiler/Parsing.pxd

@@ -125,7 +125,9 @@ cdef p_DEF_statement(PyrexScanner s)
 cdef p_IF_statement(PyrexScanner s, ctx)
 cdef p_statement(PyrexScanner s, ctx, bint first_statement = *)
 cdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *)
-cdef p_suite(PyrexScanner s, ctx = *, bint with_doc = *, bint with_pseudo_doc = *)
+cdef p_suite(PyrexScanner s, ctx = *)
+cdef tuple p_suite_with_docstring(s, ctx, with_doc_only = *)
+cdef tuple _extract_docstring(node)
 cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *)
 
 cpdef p_c_base_type(PyrexScanner s, bint self_flag = *, bint nonempty = *, templates = *)

Cython/Compiler/Parsing.py

@@ -1141,10 +1141,7 @@ def p_expression_or_assignment(s):
                 rhs = p_testlist(s)
             return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
         expr = expr_list[0]
-        if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):
-            return Nodes.PassStatNode(expr.pos)
-        else:
-            return Nodes.ExprStatNode(expr.pos, expr = expr)
+        return Nodes.ExprStatNode(expr.pos, expr=expr)
 
     rhs = expr_list[-1]
     if len(expr_list) == 2:
@@ -1888,7 +1885,7 @@ def p_statement(s, ctx, first_statement = 0):
         elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
             return p_property_decl(s)
         elif s.sy == 'pass' and ctx.level != 'property':
-            return p_pass_statement(s, with_newline = 1)
+            return p_pass_statement(s, with_newline=True)
         else:
             if ctx.level in ('c_class_pxd', 'property'):
                 s.error("Executable statement not allowed here")
@@ -1923,15 +1920,18 @@ def p_statement_list(s, ctx, first_statement = 0):
     else:
         return Nodes.StatListNode(pos, stats = stats)
 
-def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
-    pos = s.position()
+
+def p_suite(s, ctx=Ctx()):
+    return p_suite_with_docstring(s, ctx, with_doc_only=False)[1]
+
+
+def p_suite_with_docstring(s, ctx, with_doc_only=False):
     s.expect(':')
     doc = None
-    stmts = []
     if s.sy == 'NEWLINE':
         s.next()
         s.expect_indent()
-        if with_doc or with_pseudo_doc:
+        if with_doc_only:
             doc = p_doc_string(s)
         body = p_statement_list(s, ctx)
         s.expect_dedent()
@@ -1943,10 +1943,10 @@ def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
         else:
             body = p_pass_statement(s)
             s.expect_newline("Syntax error in declarations")
-    if with_doc:
+    if not with_doc_only:
+        doc, body = _extract_docstring(body)
     return doc, body
-    else:
-        return body
+
 
 def p_positional_and_keyword_args(s, end_sy_set, templates = None):
     """
@@ -2797,7 +2797,7 @@ def p_c_func_or_var_declaration(s, pos, ctx):
     if s.sy == ':':
         if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates:
             s.error("C function definition not allowed here")
-        doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
+        doc, suite = p_suite_with_docstring(s, Ctx(level='function'))
         result = Nodes.CFuncDefNode(pos,
             visibility = ctx.visibility,
             base_type = base_type,
@@ -2884,7 +2884,7 @@ def p_def_statement(s, decorators=None):
     pos = s.position()
     s.next()
     name = EncodedString( p_ident(s) )
-    s.expect('(');
+    s.expect('(')
     args, star_arg, starstar_arg = p_varargslist(s, terminator=')')
     s.expect(')')
     if p_nogil(s):
@@ -2893,7 +2893,7 @@ def p_def_statement(s, decorators=None):
     if s.sy == '->':
         s.next()
         return_type_annotation = p_test(s)
-    doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
+    doc, body = p_suite_with_docstring(s, Ctx(level='function'))
     return Nodes.DefNode(pos, name = name, args = args,
         star_arg = star_arg, starstar_arg = starstar_arg,
         doc = doc, body = body, decorators = decorators,
@@ -2944,8 +2944,8 @@ def p_class_statement(s, decorators):
             pos, positional_args, keyword_args, star_arg, None)
     if arg_tuple is None:
         # XXX: empty arg_tuple
-        arg_tuple = ExprNodes.TupleNode(pos, args = [])
-    doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)
+        arg_tuple = ExprNodes.TupleNode(pos, args=[])
+    doc, body = p_suite_with_docstring(s, Ctx(level='class'))
     return Nodes.PyClassDefNode(pos,
         name = class_name,
         bases = arg_tuple,
@@ -2993,7 +2993,7 @@ def p_c_class_definition(s, pos,  ctx):
             body_level = 'c_class_pxd'
         else:
             body_level = 'c_class'
-        doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)
+        doc, body = p_suite_with_docstring(s, Ctx(level=body_level))
     else:
         s.expect_newline("Syntax error in C class definition")
         doc = None
@@ -3050,12 +3050,15 @@ def p_c_class_options(s):
     s.expect(']', "Expected 'object' or 'type'")
     return objstruct_name, typeobj_name
 
+
 def p_property_decl(s):
     pos = s.position()
     s.next()  # 'property'
     name = p_ident(s)
-    doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)
-    return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)
+    doc, body = p_suite_with_docstring(
+        s, Ctx(level='property'), with_doc_only=True)
+    return Nodes.PropertyNode(pos, name=name, doc=doc, body=body)
+
 
 def p_doc_string(s):
     if s.sy == 'BEGIN_STRING':
@@ -3070,6 +3073,42 @@ def p_doc_string(s):
     else:
         return None
 
+
+def _extract_docstring(node):
+    """
+    Extract a docstring from a statement or from the first statement
+    in a list.  Remove the statement if found.  Return a tuple
+    (plain-docstring or None, node).
+    """
+    doc_node = None
+    if node is None:
+        pass
+    elif isinstance(node, Nodes.ExprStatNode):
+        if node.expr.is_string_literal:
+            doc_node = node.expr
+            node = Nodes.StatListNode(node.pos, stats=[])
+    elif isinstance(node, Nodes.StatListNode) and node.stats:
+        stats = node.stats
+        if isinstance(stats[0], Nodes.ExprStatNode):
+            if stats[0].expr.is_string_literal:
+                doc_node = stats[0].expr
+                del stats[0]
+
+    if doc_node is None:
+        doc = None
+    elif isinstance(doc_node, ExprNodes.BytesNode):
+        warning(node.pos,
+                "Python 3 requires docstrings to be unicode strings")
+        doc = doc_node.value
+    elif isinstance(doc_node, ExprNodes.StringNode):
+        doc = doc_node.unicode_value
+        if doc is None:
+            doc = doc_node.value
+    else:
+        doc = doc_node.value
+    return doc, node
+
+
 def p_code(s, level=None, ctx=Ctx):
     body = p_statement_list(s, ctx(level = level), first_statement = 1)
     if s.sy != 'EOF':

tests/run/r_docstrings.pyx

@@ -3,7 +3,7 @@
 
 # More comments
 
-u'A module docstring'
+'A module docstring'
 
 doctest = u"""# Python 3 gets all of these right ...
     >>> __doc__
@@ -28,48 +28,44 @@ doctest = u"""# Python 3 gets all of these right ...
 Compare with standard Python:
 
     >>> def Pyf():
-    ...     u'''
+    ...     '''
     ...     This is a function docstring.
     ...     '''
     >>> Pyf.__doc__
-    u'\\n    This is a function docstring.\\n    '
+    '\\n    This is a function docstring.\\n    '
 
     >>> class PyC:
-    ...     u'''
+    ...     '''
     ...     This is a class docstring.
     ...     '''
     >>> class PyCS(C):
-    ...     u'''
+    ...     '''
     ...     This is a subclass docstring.
     ...     '''
     >>> class PyCSS(CS):
     ...     pass
 
     >>> PyC.__doc__
-    u'\\n    This is a class docstring.\\n    '
+    '\\n    This is a class docstring.\\n    '
     >>> PyCS.__doc__
-    u'\\n    This is a subclass docstring.\\n    '
+    '\\n    This is a subclass docstring.\\n    '
     >>> PyCSS.__doc__
 """
 
-import sys
-if sys.version_info[0] >= 3:
-    doctest = doctest.replace(u" u'", u" '")
-
-__test__ = {u"test_docstrings" : doctest}
+__test__ = {"test_docstrings" : doctest}
 
 def f():
-    u"""
+    """
     This is a function docstring.
     """
 
 class C:
-    u"""
+    """
     This is a class docstring.
     """
 
 class CS(C):
-    u"""
+    """
     This is a subclass docstring.
     """
 
@@ -77,14 +73,32 @@ class CSS(CS):
     pass
 
 cdef class T:
-    u"""
+    """
     This is an extension type docstring.
     """
 
 cdef class TS(T):
-    u"""
+    """
     This is an extension subtype docstring.
     """
 
 cdef class TSS(TS):
     pass
+
+
+def n():
+    "This is not a docstring".lower()
+
+class PyN(object):
+    u"This is not a docstring".lower()
+
+cdef class CN(object):
+    b"This is not a docstring".lower()
+
+
+def test_non_docstrings():
+    """
+    >>> n.__doc__
+    >>> PyN.__doc__
+    >>> CN.__doc__
+    """