bpo-32911: Revert bpo-29463. (GH-7121) (GH-7197)

Remove the docstring attribute of AST types and restore docstring
expression as a first stmt in their body.
Co-authored-by: INADA Naoki <methane@users.noreply.github.com>

Doc/library/ast.rst

@@ -151,10 +151,6 @@ and classes for traversing abstract syntax trees:
    .. versionchanged:: 3.5
       :class:`AsyncFunctionDef` is now supported.
 
-   .. versionchanged:: 3.7
-      The docstring is now exported from the node docstring field, instead of
-      the first body statement.
-
 
 .. function:: fix_missing_locations(node)
 

Doc/whatsnew/3.7.rst

@@ -2161,13 +2161,6 @@ Changes in Python Behavior
 Changes in the Python API
 -------------------------
 
-* ``Module``, ``FunctionDef``, ``AsyncFunctionDef``, and
-  ``ClassDef`` AST nodes now have the new ``docstring`` attribute.
-  The first statement in their body is not considered as a docstring
-  anymore.  ``co_firstlineno`` and ``co_lnotab`` of code object for class
-  and module are affected by this change. (Contributed by INADA Naoki and
-  Eugene Toder in :issue:`29463`.)
-
 * :meth:`socketserver.ThreadingMixIn.server_close` now waits until all
   non-daemon threads complete.  Set the new
   :attr:`socketserver.ThreadingMixIn.block_on_close` class attribute to

Include/Python-ast.h

@@ -46,7 +46,6 @@ struct _mod {
     union {
         struct {
             asdl_seq *body;
-            string docstring;
         } Module;
 
         struct {
@@ -81,7 +80,6 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
             expr_ty returns;
-            string docstring;
         } FunctionDef;
 
         struct {
@@ -90,7 +88,6 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
             expr_ty returns;
-            string docstring;
         } AsyncFunctionDef;
 
         struct {
@@ -99,7 +96,6 @@ struct _stmt {
             asdl_seq *keywords;
             asdl_seq *body;
             asdl_seq *decorator_list;
-            string docstring;
         } ClassDef;
 
         struct {
@@ -443,27 +439,26 @@ struct _withitem {
 };
 
 
-#define Module(a0, a1, a2) _Py_Module(a0, a1, a2)
-mod_ty _Py_Module(asdl_seq * body, string docstring, PyArena *arena);
+#define Module(a0, a1) _Py_Module(a0, a1)
+mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
 #define Interactive(a0, a1) _Py_Interactive(a0, a1)
 mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
 #define Expression(a0, a1) _Py_Expression(a0, a1)
 mod_ty _Py_Expression(expr_ty body, PyArena *arena);
 #define Suite(a0, a1) _Py_Suite(a0, a1)
 mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
-#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
+#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
-                        asdl_seq * decorator_list, expr_ty returns, string
-                        docstring, int lineno, int col_offset, PyArena *arena);
-#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
+                        asdl_seq * decorator_list, expr_ty returns, int lineno,
+                        int col_offset, PyArena *arena);
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
                              body, asdl_seq * decorator_list, expr_ty returns,
-                             string docstring, int lineno, int col_offset,
-                             PyArena *arena);
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
+                             int lineno, int col_offset, PyArena *arena);
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
-                     asdl_seq * body, asdl_seq * decorator_list, string
-                     docstring, int lineno, int col_offset, PyArena *arena);
+                     asdl_seq * body, asdl_seq * decorator_list, int lineno,
+                     int col_offset, PyArena *arena);
 #define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
 stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)

Lib/ast.py

@@ -206,7 +206,15 @@ def get_docstring(node, clean=True):
     """
     if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
         raise TypeError("%r can't have docstrings" % node.__class__.__name__)
-    text = node.docstring
+    if not node.body:
+        return None
+    node = node.body[0].value
+    if isinstance(node, Str):
+        text = node.s
+    elif isinstance(node, Constant) and isinstance(node.value, str):
+        text = node.value
+    else:
+        return None
     if clean and text:
         import inspect
         text = inspect.cleandoc(text)

Lib/test/test_opcodes.py

@@ -27,7 +27,7 @@ class OpcodeTest(unittest.TestCase):
             with open(ann_module.__file__) as f:
                 txt = f.read()
             co = compile(txt, ann_module.__file__, 'exec')
-            self.assertEqual(co.co_firstlineno, 8)
+            self.assertEqual(co.co_firstlineno, 6)
         except OSError:
             pass
 

Misc/NEWS.d/3.7.0a1.rst

@@ -788,7 +788,7 @@ objects.
 Add ``docstring`` field to Module, ClassDef, FunctionDef, and
 AsyncFunctionDef ast nodes.  docstring is not first stmt in their body
 anymore.  It affects ``co_firstlineno`` and ``co_lnotab`` of code object for
-module and class.
+module and class. (Reverted in :issue:`32911`.)
 
 ..
 

Misc/NEWS.d/next/Core and Builtins/2018-02-27-20-57-00.bpo-32911.cmKfco.rst

+Due to unexpected compatibility issues discovered during downstream beta
+testing, reverted :issue:`29463`. ``docstring`` field is removed from Module,
+ClassDef, FunctionDef, and AsyncFunctionDef ast nodes which was added in
+3.7a1.  Docstring expression is restored as a first statement in their body.
+Based on patch by Inada Naoki.

Parser/Python.asdl

@@ -6,7 +6,7 @@
 
 module Python
 {
-    mod = Module(stmt* body, string? docstring)
+    mod = Module(stmt* body)
         | Interactive(stmt* body)
         | Expression(expr body)
 
@@ -14,18 +14,15 @@ module Python
         | Suite(stmt* body)
 
     stmt = FunctionDef(identifier name, arguments args,
-                       stmt* body, expr* decorator_list, expr? returns,
-                       string? docstring)
+                       stmt* body, expr* decorator_list, expr? returns)
           | AsyncFunctionDef(identifier name, arguments args,
-                             stmt* body, expr* decorator_list, expr? returns,
-                             string? docstring)
+                             stmt* body, expr* decorator_list, expr? returns)
 
           | ClassDef(identifier name,
              expr* bases,
              keyword* keywords,
              stmt* body,
-             expr* decorator_list,
-             string? docstring)
+             expr* decorator_list)
           | Return(expr? value)
 
           | Delete(expr* targets)

Python/ast_opt.c

@@ -467,12 +467,51 @@ static int astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, int opti
     } \
 }
 
+static int
+isdocstring(stmt_ty s)
+{
+    if (s->kind != Expr_kind)
+        return 0;
+    if (s->v.Expr.value->kind == Str_kind)
+        return 1;
+    if (s->v.Expr.value->kind == Constant_kind)
+        return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
+    return 0;
+}
+
+static int
+astfold_body(asdl_seq *stmts, PyArena *ctx_, int optimize_)
+{
+    if (!asdl_seq_LEN(stmts)) {
+        return 1;
+    }
+    int docstring = isdocstring((stmt_ty)asdl_seq_GET(stmts, 0));
+    CALL_SEQ(astfold_stmt, stmt_ty, stmts);
+    if (docstring) {
+        return 1;
+    }
+    stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
+    if (isdocstring(st)) {
+        asdl_seq *values = _Py_asdl_seq_new(1, ctx_);
+        if (!values) {
+            return 0;
+        }
+        asdl_seq_SET(values, 0, st->v.Expr.value);
+        expr_ty expr = _Py_JoinedStr(values, st->lineno, st->col_offset, ctx_);
+        if (!expr) {
+            return 0;
+        }
+        st->v.Expr.value = expr;
+    }
+    return 1;
+}
+
 static int
 astfold_mod(mod_ty node_, PyArena *ctx_, int optimize_)
 {
     switch (node_->kind) {
     case Module_kind:
-        CALL_SEQ(astfold_stmt, stmt_ty, node_->v.Module.body);
+        CALL(astfold_body, asdl_seq, node_->v.Module.body);
         break;
     case Interactive_kind:
         CALL_SEQ(astfold_stmt, stmt_ty, node_->v.Interactive.body);
@@ -657,20 +696,20 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, int optimize_)
     switch (node_->kind) {
     case FunctionDef_kind:
         CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args);
-        CALL_SEQ(astfold_stmt, stmt_ty, node_->v.FunctionDef.body);
+        CALL(astfold_body, asdl_seq, node_->v.FunctionDef.body);
         CALL_SEQ(astfold_expr, expr_ty, node_->v.FunctionDef.decorator_list);
         CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns);
         break;
     case AsyncFunctionDef_kind:
         CALL(astfold_arguments, arguments_ty, node_->v.AsyncFunctionDef.args);
-        CALL_SEQ(astfold_stmt, stmt_ty, node_->v.AsyncFunctionDef.body);
+        CALL(astfold_body, asdl_seq, node_->v.AsyncFunctionDef.body);
         CALL_SEQ(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.decorator_list);
         CALL_OPT(astfold_expr, expr_ty, node_->v.AsyncFunctionDef.returns);
         break;
     case ClassDef_kind:
         CALL_SEQ(astfold_expr, expr_ty, node_->v.ClassDef.bases);
         CALL_SEQ(astfold_keyword, keyword_ty, node_->v.ClassDef.keywords);
-        CALL_SEQ(astfold_stmt, stmt_ty, node_->v.ClassDef.body);
+        CALL(astfold_body, asdl_seq, node_->v.ClassDef.body);
         CALL_SEQ(astfold_expr, expr_ty, node_->v.ClassDef.decorator_list);
         break;
     case Return_kind:

Python/compile.c

@@ -1392,6 +1392,18 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
     } \
 }
 
+static int
+compiler_isdocstring(stmt_ty s)
+{
+    if (s->kind != Expr_kind)
+        return 0;
+    if (s->v.Expr.value->kind == Str_kind)
+        return 1;
+    if (s->v.Expr.value->kind == Constant_kind)
+        return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
+    return 0;
+}
+
 static int
 is_const(expr_ty e)
 {
@@ -1587,27 +1599,37 @@ compiler_unwind_fblock(struct compiler *c, struct fblockinfo *info,
    and for annotations. */
 
 static int
-compiler_body(struct compiler *c, asdl_seq *stmts, string docstring)
+compiler_body(struct compiler *c, asdl_seq *stmts)
 {
+    int i = 0;
+    stmt_ty st;
+
     /* Set current line number to the line number of first statement.
        This way line number for SETUP_ANNOTATIONS will always
        coincide with the line number of first "real" statement in module.
        If body is empy, then lineno will be set later in assemble. */
     if (c->u->u_scope_type == COMPILER_SCOPE_MODULE &&
         !c->u->u_lineno && asdl_seq_LEN(stmts)) {
-        stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
+        st = (stmt_ty)asdl_seq_GET(stmts, 0);
         c->u->u_lineno = st->lineno;
     }
     /* Every annotated class and module should have __annotations__. */
     if (find_ann(stmts)) {
         ADDOP(c, SETUP_ANNOTATIONS);
     }
+    if (!asdl_seq_LEN(stmts))
+        return 1;
+    st = (stmt_ty)asdl_seq_GET(stmts, 0);
     /* if not -OO mode, set docstring */
-    if (c->c_optimize < 2 && docstring) {
-        ADDOP_LOAD_CONST(c, docstring);
-        ADDOP_NAME(c, STORE_NAME, __doc__, names);
+    if (compiler_isdocstring(st) && c->c_optimize < 2) {
+        /* don't generate docstrings if -OO */
+        i = 1;
+        VISIT(c, expr, st->v.Expr.value);
+        if (!compiler_nameop(c, __doc__, Store))
+            return 0;
     }
-    VISIT_SEQ(c, stmt, stmts);
+    for (; i < asdl_seq_LEN(stmts); i++)
+        VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
     return 1;
 }
 
@@ -1627,7 +1649,7 @@ compiler_mod(struct compiler *c, mod_ty mod)
         return NULL;
     switch (mod->kind) {
     case Module_kind:
-        if (!compiler_body(c, mod->v.Module.body, mod->v.Module.docstring)) {
+        if (!compiler_body(c, mod->v.Module.body)) {
             compiler_exit_scope(c);
             return 0;
         }
@@ -1957,13 +1979,15 @@ static int
 compiler_function(struct compiler *c, stmt_ty s, int is_async)
 {
     PyCodeObject *co;
-    PyObject *qualname, *docstring = Py_None;
+    PyObject *qualname, *first_const = Py_None;
     arguments_ty args;
     expr_ty returns;
     identifier name;
     asdl_seq* decos;
     asdl_seq *body;
+    stmt_ty st;
     Py_ssize_t i, funcflags;
+    int docstring;
     int annotations;
     int scope_type;
 
@@ -2010,16 +2034,21 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
     }
 
     /* if not -OO mode, add docstring */
-    if (c->c_optimize < 2 && s->v.FunctionDef.docstring)
-        docstring = s->v.FunctionDef.docstring;
-    if (compiler_add_const(c, docstring) < 0) {
+    st = (stmt_ty)asdl_seq_GET(body, 0);
+    docstring = compiler_isdocstring(st);
+    if (docstring && c->c_optimize < 2) {
+        if (st->v.Expr.value->kind == Constant_kind)
+            first_const = st->v.Expr.value->v.Constant.value;
+        else
+            first_const = st->v.Expr.value->v.Str.s;
+    }
+    if (compiler_add_const(c, first_const) < 0) {
         compiler_exit_scope(c);
         return 0;
     }
 
     c->u->u_argcount = asdl_seq_LEN(args->args);
     c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
-    /* if there was a docstring, we need to skip the first statement */
     VISIT_SEQ_IN_SCOPE(c, stmt, body);
     co = assemble(c, 1);
     qualname = c->u->u_qualname;
@@ -2101,7 +2130,7 @@ compiler_class(struct compiler *c, stmt_ty s)
         }
         Py_DECREF(str);
         /* compile the body proper */
-        if (!compiler_body(c, s->v.ClassDef.body, s->v.ClassDef.docstring)) {
+        if (!compiler_body(c, s->v.ClassDef.body)) {
             compiler_exit_scope(c);
             return 0;
         }

Python/future.c

@@ -63,6 +63,7 @@ static int
 future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
 {
     int i, done = 0, prev_line = 0;
+    stmt_ty first;
 
     if (!(mod->kind == Module_kind || mod->kind == Interactive_kind))
         return 1;
@@ -78,7 +79,15 @@ future_parse(PyFutureFeatures *ff, mod_ty mod, PyObject *filename)
        but is preceded by a regular import.
     */
 
-    for (i = 0; i < asdl_seq_LEN(mod->v.Module.body); i++) {
+    i = 0;
+    first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
+    if (first->kind == Expr_kind
+        && (first->v.Expr.value->kind == Str_kind
+            || (first->v.Expr.value->kind == Constant_kind
+                && PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value))))
+        i++;
+
+    for (; i < asdl_seq_LEN(mod->v.Module.body); i++) {
         stmt_ty s = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
 
         if (done && s->lineno > prev_line)

Tools/parser/unparse.py

@@ -71,8 +71,6 @@ class Unparser:
     ########################################################
 
     def _Module(self, tree):
-        if tree.docstring is not None:
-            self.fill(repr(tree.docstring))
         for stmt in tree.body:
             self.dispatch(stmt)
 
@@ -237,8 +235,6 @@ class Unparser:
         self.write(")")
 
         self.enter()
-        if t.docstring is not None:
-            self.fill(repr(t.docstring))
         self.dispatch(t.body)
         self.leave()
 
@@ -261,8 +257,6 @@ class Unparser:
             self.write(" -> ")
             self.dispatch(t.returns)
         self.enter()
-        if t.docstring is not None:
-            self.fill(repr(t.docstring))
         self.dispatch(t.body)
         self.leave()