Issue #24254: Preserve class attribute definition order.

Doc/library/types.rst

@@ -53,8 +53,20 @@ Dynamic Type Creation
    in *kwds* argument with any ``'metaclass'`` entry removed. If no *kwds*
    argument is passed in, this will be an empty dict.
 
+   .. impl-detail::
+
+      CPython uses :class:`collections.OrderedDict` for the default
+      namespace.
+
    .. versionadded:: 3.3
 
+   .. versionchanged:: 3.6
+
+      The default value for the ``namespace`` element of the returned
+      tuple has changed from :func:`dict`.  Now an insertion-order-
+      preserving mapping is used when the metaclass does not have a
+      ``__prepare__`` method,
+
 .. seealso::
 
    :ref:`metaclasses`

Doc/reference/compound_stmts.rst

@@ -632,6 +632,17 @@ list for the base classes and the saved local namespace for the attribute
 dictionary.  The class name is bound to this class object in the original local
 namespace.
 
+The order in which attributes are defined in the class body is preserved
+in the ``__definition_order__`` attribute on the new class.  If that order
+is not known then the attribute is set to :const:`None`.  The class body
+may include a ``__definition_order__`` attribute.  In that case it is used
+directly.  The value must be a tuple of identifiers or ``None``, otherwise
+:exc:`TypeError` will be raised when the class statement is executed.
+
+.. versionchanged:: 3.6
+
+   Add ``__definition_order__`` to classes.
+
 Class creation can be customized heavily using :ref:`metaclasses <metaclasses>`.
 
 Classes can also be decorated: just like when decorating functions, ::

Doc/reference/datamodel.rst

@@ -1750,7 +1750,14 @@ as ``namespace = metaclass.__prepare__(name, bases, **kwds)`` (where the
 additional keyword arguments, if any, come from the class definition).
 
 If the metaclass has no ``__prepare__`` attribute, then the class namespace
-is initialised as an empty :func:`dict` instance.
+is initialised as an empty ordered mapping.
+
+.. impl-detail::
+
+   In CPython the default is :class:`collections.OrderedDict`.
+
+.. versionchanged:: 3.6
+   Defaults to :class:`collections.OrderedDict` instead of :func:`dict`.
 
 .. seealso::
 

Doc/whatsnew/3.6.rst

@@ -92,6 +92,7 @@ Windows improvements:
     :pep:`4XX` - Python Virtual Environments
        PEP written by Carl Meyer
 
+.. XXX PEP 520: :ref:`Preserving Class Attribute Definition Order<whatsnew-deforder>`
 
 New Features
 ============
@@ -271,6 +272,31 @@ Example of fatal error on buffer overflow using
 (Contributed by Victor Stinner in :issue:`26516` and :issue:`26564`.)
 
 
+.. _whatsnew-deforder:
+
+PEP 520: Preserving Class Attribute Definition Order
+----------------------------------------------------
+
+Attributes in a class definition body have a natural ordering: the same
+order in which the names appear in the source.  This order is now
+preserved in the new class's ``__definition_order__`` attribute.  It is
+a tuple of the attribute names, in the order in which they appear in
+the class definition body.
+
+For types that don't have a definition (e.g. builtins), or the attribute
+order could not be determined, ``__definition_order__`` is ``None``.
+
+Also, the effective default class *execution* namespace (returned from
+``type.__prepare__()``) is now an insertion-order-preserving mapping.
+For CPython, it is now ``collections.OrderedDict``.  Note that the
+class namespace, ``cls.__dict__``, is unchanged.
+
+.. seealso::
+
+   :pep:`520` - Preserving Class Attribute Definition Order
+      PEP written and implemented by Eric Snow.
+
+
 Other Language Changes
 ======================
 

Include/object.h

@@ -421,6 +421,8 @@ typedef struct _typeobject {
 
     destructor tp_finalize;
 
+    PyObject *tp_deforder;
+
 #ifdef COUNT_ALLOCS
     /* these must be last and never explicitly initialized */
     Py_ssize_t tp_allocs;

Include/odictobject.h

@@ -28,6 +28,10 @@ PyAPI_FUNC(PyObject *) PyODict_New(void);
 PyAPI_FUNC(int) PyODict_SetItem(PyObject *od, PyObject *key, PyObject *item);
 PyAPI_FUNC(int) PyODict_DelItem(PyObject *od, PyObject *key);
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyODict_KeysAsTuple(PyObject *od);
+#endif
+
 /* wrappers around PyDict* functions */
 #define PyODict_GetItem(od, key) PyDict_GetItem((PyObject *)od, key)
 #define PyODict_GetItemWithError(od, key) \

Lib/test/test_builtin.py

@@ -16,8 +16,10 @@ import traceback
 import types
 import unittest
 import warnings
+from collections import OrderedDict
 from operator import neg
-from test.support import TESTFN, unlink,  run_unittest, check_warnings
+from test.support import (TESTFN, unlink,  run_unittest, check_warnings,
+                          cpython_only)
 from test.support.script_helper import assert_python_ok
 try:
     import pty, signal
@@ -1778,6 +1780,194 @@ class TestType(unittest.TestCase):
             A.__doc__ = doc
             self.assertEqual(A.__doc__, doc)
 
+    def test_type_definition_order_nonempty(self):
+        class Spam:
+            b = 1
+            c = 3
+            a = 2
+            d = 4
+            eggs = 2
+            e = 5
+            b = 42
+
+        self.assertEqual(Spam.__definition_order__,
+                         ('__module__', '__qualname__',
+                          'b', 'c', 'a', 'd', 'eggs', 'e'))
+
+    def test_type_definition_order_empty(self):
+        class Empty:
+            pass
+
+        self.assertEqual(Empty.__definition_order__,
+                         ('__module__', '__qualname__'))
+
+    def test_type_definition_order_on_instance(self):
+        class Spam:
+            a = 2
+            b = 1
+            c = 3
+        with self.assertRaises(AttributeError):
+            Spam().__definition_order__
+
+    def test_type_definition_order_set_to_None(self):
+        class Spam:
+            a = 2
+            b = 1
+            c = 3
+        Spam.__definition_order__ = None
+        self.assertEqual(Spam.__definition_order__, None)
+
+    def test_type_definition_order_set_to_tuple(self):
+        class Spam:
+            a = 2
+            b = 1
+            c = 3
+        Spam.__definition_order__ = ('x', 'y', 'z')
+        self.assertEqual(Spam.__definition_order__, ('x', 'y', 'z'))
+
+    def test_type_definition_order_deleted(self):
+        class Spam:
+            a = 2
+            b = 1
+            c = 3
+        del Spam.__definition_order__
+        self.assertEqual(Spam.__definition_order__, None)
+
+    def test_type_definition_order_set_to_bad_type(self):
+        class Spam:
+            a = 2
+            b = 1
+            c = 3
+        Spam.__definition_order__ = 42
+        self.assertEqual(Spam.__definition_order__, 42)
+
+    def test_type_definition_order_builtins(self):
+        self.assertEqual(object.__definition_order__, None)
+        self.assertEqual(type.__definition_order__, None)
+        self.assertEqual(dict.__definition_order__, None)
+        self.assertEqual(type(None).__definition_order__, None)
+
+    def test_type_definition_order_dunder_names_included(self):
+        class Dunder:
+            VAR = 3
+            def __init__(self):
+                pass
+
+        self.assertEqual(Dunder.__definition_order__,
+                         ('__module__', '__qualname__',
+                          'VAR', '__init__'))
+
+    def test_type_definition_order_only_dunder_names(self):
+        class DunderOnly:
+            __xyz__ = None
+            def __init__(self):
+                pass
+
+        self.assertEqual(DunderOnly.__definition_order__,
+                         ('__module__', '__qualname__',
+                          '__xyz__', '__init__'))
+
+    def test_type_definition_order_underscore_names(self):
+        class HalfDunder:
+            __whether_to_be = True
+            or_not_to_be__ = False
+
+        self.assertEqual(HalfDunder.__definition_order__,
+                         ('__module__', '__qualname__',
+                          '_HalfDunder__whether_to_be', 'or_not_to_be__'))
+
+    def test_type_definition_order_with_slots(self):
+        class Slots:
+            __slots__ = ('x', 'y')
+            a = 1
+            b = 2
+
+        self.assertEqual(Slots.__definition_order__,
+                         ('__module__', '__qualname__',
+                          '__slots__', 'a', 'b'))
+
+    def test_type_definition_order_overwritten_None(self):
+        class OverwrittenNone:
+            __definition_order__ = None
+            a = 1
+            b = 2
+            c = 3
+
+        self.assertEqual(OverwrittenNone.__definition_order__, None)
+
+    def test_type_definition_order_overwritten_tuple(self):
+        class OverwrittenTuple:
+            __definition_order__ = ('x', 'y', 'z')
+            a = 1
+            b = 2
+            c = 3
+
+        self.assertEqual(OverwrittenTuple.__definition_order__,
+                         ('x', 'y', 'z'))
+
+    def test_type_definition_order_overwritten_bad_item(self):
+        with self.assertRaises(TypeError):
+            class PoorlyOverwritten:
+                __definition_order__ = ('a', 2, 'c')
+                a = 1
+                b = 2
+                c = 3
+
+    def test_type_definition_order_overwritten_bad_type(self):
+        with self.assertRaises(TypeError):
+            class PoorlyOverwritten:
+                __definition_order__ = ['a', 2, 'c']
+                a = 1
+                b = 2
+                c = 3
+
+    def test_type_definition_order_metaclass(self):
+        class Meta(type):
+            SPAM = 42
+
+            def __init__(self, *args, **kwargs):
+                super().__init__(*args, **kwargs)
+
+        self.assertEqual(Meta.__definition_order__,
+                         ('__module__', '__qualname__',
+                          'SPAM', '__init__'))
+
+    def test_type_definition_order_OrderedDict(self):
+        class Meta(type):
+            def __prepare__(self, *args, **kwargs):
+                return OrderedDict()
+
+        class WithODict(metaclass=Meta):
+            x='y'
+
+        self.assertEqual(WithODict.__definition_order__,
+                         ('__module__', '__qualname__', 'x'))
+
+        class Meta(type):
+            def __prepare__(self, *args, **kwargs):
+                class ODictSub(OrderedDict):
+                    pass
+                return ODictSub()
+
+        class WithODictSub(metaclass=Meta):
+            x='y'
+
+        self.assertEqual(WithODictSub.__definition_order__,
+                         ('__module__', '__qualname__', 'x'))
+
+    @cpython_only
+    def test_type_definition_order_cpython(self):
+        # some implementations will have an ordered-by-default dict.
+
+        class Meta(type):
+            def __prepare__(self, *args, **kwargs):
+                return {}
+
+        class NotOrdered(metaclass=Meta):
+            x='y'
+
+        self.assertEqual(NotOrdered.__definition_order__, None)
+
     def test_bad_args(self):
         with self.assertRaises(TypeError):
             type()

Lib/test/test_metaclass.py

@@ -180,7 +180,7 @@ Use a metaclass that doesn't derive from type.
     meta: C ()
     ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)]
     kw: []
-    >>> type(C) is dict
+    >>> type(C) is types._DefaultClassNamespaceType
     True
     >>> print(sorted(C.items()))
     [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('a', 42), ('b', 24)]
@@ -211,8 +211,11 @@ And again, with a __prepare__ attribute.
 
 The default metaclass must define a __prepare__() method.
 
-    >>> type.__prepare__()
-    {}
+    >>> ns = type.__prepare__()
+    >>> type(ns) is types._DefaultClassNamespaceType
+    True
+    >>> list(ns) == []
+    True
     >>>
 
 Make sure it works with subclassing.
@@ -248,7 +251,9 @@ Test failures in looking up the __prepare__ method work.
 
 """
 
+from collections import OrderedDict
 import sys
+import types
 
 # Trace function introduces __locals__ which causes various tests to fail.
 if hasattr(sys, 'gettrace') and sys.gettrace():

Lib/test/test_pydoc.py

@@ -427,6 +427,7 @@ class PydocDocTest(unittest.TestCase):
         expected_html = expected_html_pattern % (
                         (mod_url, mod_file, doc_loc) +
                         expected_html_data_docstrings)
+        self.maxDiff = None
         self.assertEqual(result, expected_html)
 
     @unittest.skipIf(sys.flags.optimize >= 2,
@@ -473,13 +474,18 @@ class PydocDocTest(unittest.TestCase):
     def test_non_str_name(self):
         # issue14638
         # Treat illegal (non-str) name like no name
+        # Definition order is set to None so it looks the same in both
+        # cases.
         class A:
+            __definition_order__ = None
             __name__ = 42
         class B:
             pass
         adoc = pydoc.render_doc(A())
         bdoc = pydoc.render_doc(B())
-        self.assertEqual(adoc.replace("A", "B"), bdoc)
+        self.maxDiff = None
+        expected = adoc.replace("A", "B")
+        self.assertEqual(bdoc, expected)
 
     def test_not_here(self):
         missing_module = "test.i_am_not_here"

Lib/test/test_sys.py

@@ -1084,7 +1084,7 @@ class SizeofTest(unittest.TestCase):
         check((1,2,3), vsize('') + 3*self.P)
         # type
         # static type: PyTypeObject
-        fmt = 'P2n15Pl4Pn9Pn11PIP'
+        fmt = 'P2n15Pl4Pn9Pn11PIPP'
         if hasattr(sys, 'getcounts'):
             fmt += '3n2P'
         s = vsize(fmt)

Lib/test/test_types.py

@@ -825,6 +825,28 @@ class ClassCreationTests(unittest.TestCase):
         self.assertEqual(C.y, 1)
         self.assertEqual(C.z, 2)
 
+    def test_new_class_deforder(self):
+        C = types.new_class("C")
+        self.assertEqual(C.__definition_order__, tuple())
+
+        Meta = self.Meta
+        def func(ns):
+            ns["x"] = 0
+        D = types.new_class("D", (), {"metaclass": Meta, "z": 2}, func)
+        self.assertEqual(D.__definition_order__, ('y', 'z', 'x'))
+
+        def func(ns):
+            ns["__definition_order__"] = None
+            ns["x"] = 0
+        D = types.new_class("D", (), {"metaclass": Meta, "z": 2}, func)
+        self.assertEqual(D.__definition_order__, None)
+
+        def func(ns):
+            ns["__definition_order__"] = ('a', 'b', 'c')
+            ns["x"] = 0
+        D = types.new_class("D", (), {"metaclass": Meta, "z": 2}, func)
+        self.assertEqual(D.__definition_order__, ('a', 'b', 'c'))
+
     # Many of the following tests are derived from test_descr.py
     def test_prepare_class(self):
         # Basic test of metaclass derivation

Lib/types.py

@@ -25,8 +25,11 @@ CoroutineType = type(_c)
 _c.close()  # Prevent ResourceWarning
 
 class _C:
+    _nsType = type(locals())
     def _m(self): pass
 MethodType = type(_C()._m)
+# In CPython, this should end up as OrderedDict.
+_DefaultClassNamespaceType = _C._nsType
 
 BuiltinFunctionType = type(len)
 BuiltinMethodType = type([].append)     # Same as BuiltinFunctionType
@@ -85,7 +88,7 @@ def prepare_class(name, bases=(), kwds=None):
     if hasattr(meta, '__prepare__'):
         ns = meta.__prepare__(name, bases, **kwds)
     else:
-        ns = {}
+        ns = _DefaultClassNamespaceType()
     return meta, ns, kwds
 
 def _calculate_meta(meta, bases):

Lib/typing.py

@@ -1301,6 +1301,7 @@ class _ProtocolMeta(GenericMeta):
                     if (not attr.startswith('_abc_') and
                             attr != '__abstractmethods__' and
                             attr != '_is_protocol' and
+                            attr != '__definition_order__' and
                             attr != '__dict__' and
                             attr != '__args__' and
                             attr != '__slots__' and

Misc/NEWS

@@ -49,6 +49,8 @@ Core and Builtins
   potentially have caused off-by-one-ulp results on platforms with
   unreliable ldexp implementations.
 
+- Issue #24254: Make class definition namespace ordered by default.
+
 - Issue #27662: Fix an overflow check in ``List_New``: the original code was
   checking against ``Py_SIZE_MAX`` instead of the correct upper bound of
   ``Py_SSIZE_T_MAX``. Patch by Xiang Zhang.

Objects/odictobject.c

@@ -1762,6 +1762,21 @@ PyODict_DelItem(PyObject *od, PyObject *key)
     return _PyDict_DelItem_KnownHash(od, key, hash);
 }
 
+PyObject *
+_PyODict_KeysAsTuple(PyObject *od) {
+    Py_ssize_t i = 0;
+    _ODictNode *node;
+    PyObject *keys = PyTuple_New(PyODict_Size(od));
+    if (keys == NULL)
+        return NULL;
+    _odict_FOREACH((PyODictObject *)od, node) {
+        Py_INCREF(_odictnode_KEY(node));
+        PyTuple_SET_ITEM(keys, i, _odictnode_KEY(node));
+        i++;
+    }
+    return keys;
+}
+
 
 /* -------------------------------------------
  * The OrderedDict views (keys/values/items)

Objects/typeobject.c

@@ -48,6 +48,7 @@ static size_t method_cache_collisions = 0;
 _Py_IDENTIFIER(__abstractmethods__);
 _Py_IDENTIFIER(__class__);
 _Py_IDENTIFIER(__delitem__);
+_Py_IDENTIFIER(__definition_order__);
 _Py_IDENTIFIER(__dict__);
 _Py_IDENTIFIER(__doc__);
 _Py_IDENTIFIER(__getattribute__);
@@ -488,6 +489,23 @@ type_set_module(PyTypeObject *type, PyObject *value, void *context)
     return _PyDict_SetItemId(type->tp_dict, &PyId___module__, value);
 }
 
+static PyObject *
+type_deforder(PyTypeObject *type, void *context)
+{
+    if (type->tp_deforder == NULL)
+        Py_RETURN_NONE;
+    Py_INCREF(type->tp_deforder);
+    return type->tp_deforder;
+}
+
+static int
+type_set_deforder(PyTypeObject *type, PyObject *value, void *context)
+{
+    Py_XINCREF(value);
+    Py_XSETREF(type->tp_deforder, value);
+    return 0;
+}
+
 static PyObject *
 type_abstractmethods(PyTypeObject *type, void *context)
 {
@@ -834,6 +852,8 @@ static PyGetSetDef type_getsets[] = {
     {"__qualname__", (getter)type_qualname, (setter)type_set_qualname, NULL},
     {"__bases__", (getter)type_get_bases, (setter)type_set_bases, NULL},
     {"__module__", (getter)type_module, (setter)type_set_module, NULL},
+    {"__definition_order__", (getter)type_deforder,
+     (setter)type_set_deforder, NULL},
     {"__abstractmethods__", (getter)type_abstractmethods,
      (setter)type_set_abstractmethods, NULL},
     {"__dict__",  (getter)type_dict,  NULL, NULL},
@@ -2351,6 +2371,7 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
         goto error;
     }
 
+    /* Copy the definition namespace into a new dict. */
     dict = PyDict_Copy(orig_dict);
     if (dict == NULL)
         goto error;
@@ -2559,6 +2580,48 @@ type_new(PyTypeObject *metatype, PyObject *args, PyObject *kwds)
     if (qualname != NULL && PyDict_DelItem(dict, PyId___qualname__.object) < 0)
         goto error;
 
+    /* Set tp_deforder to the extracted definition order, if any. */
+    type->tp_deforder = _PyDict_GetItemId(dict, &PyId___definition_order__);
+    if (type->tp_deforder != NULL) {
+        Py_INCREF(type->tp_deforder);
+
+        // Due to subclass lookup, __definition_order__ can't be in __dict__.
+        if (_PyDict_DelItemId(dict, &PyId___definition_order__) != 0) {
+            goto error;
+        }
+
+        if (type->tp_deforder != Py_None) {
+            Py_ssize_t numnames;
+
+            if (!PyTuple_Check(type->tp_deforder)) {
+                PyErr_SetString(PyExc_TypeError,
+                                "__definition_order__ must be a tuple or None");
+                goto error;
+            }
+
+            // Make sure they are identifers.
+            numnames = PyTuple_Size(type->tp_deforder);
+            for (i = 0; i < numnames; i++) {
+                PyObject *name = PyTuple_GET_ITEM(type->tp_deforder, i);
+                if (name == NULL) {
+                    goto error;
+                }
+                if (!PyUnicode_Check(name) || !PyUnicode_IsIdentifier(name)) {
+                    PyErr_Format(PyExc_TypeError,
+                                 "__definition_order__ must "
+                                 "contain only identifiers, got '%s'",
+                                 name);
+                    goto error;
+                }
+            }
+        }
+    }
+    else if (PyODict_Check(orig_dict)) {
+        type->tp_deforder = _PyODict_KeysAsTuple(orig_dict);
+        if (type->tp_deforder == NULL)
+            goto error;
+    }
+
     /* Set tp_doc to a copy of dict['__doc__'], if the latter is there
        and is a string.  The __doc__ accessor will first look for tp_doc;
        if that fails, it will still look into __dict__.
@@ -3073,6 +3136,7 @@ type_dealloc(PyTypeObject *type)
     Py_XDECREF(type->tp_mro);
     Py_XDECREF(type->tp_cache);
     Py_XDECREF(type->tp_subclasses);
+    Py_XDECREF(type->tp_deforder);
     /* A type's tp_doc is heap allocated, unlike the tp_doc slots
      * of most other objects.  It's okay to cast it to char *.
      */
@@ -3115,7 +3179,7 @@ type_subclasses(PyTypeObject *type, PyObject *args_ignored)
 static PyObject *
 type_prepare(PyObject *self, PyObject *args, PyObject *kwds)
 {
-    return PyDict_New();
+    return PyODict_New();
 }
 
 /*

Python/bltinmodule.c

@@ -145,7 +145,7 @@ builtin___build_class__(PyObject *self, PyObject *args, PyObject *kwds)
     if (prep == NULL) {
         if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
             PyErr_Clear();
-            ns = PyDict_New();
+            ns = PyODict_New();
         }
         else {
             Py_DECREF(meta);