Fixes for shared 2.6 code that implements PEP 3101, advanced string

formatting.

Includes:
 - Modifying tests for basic types to use __format__ methods, instead
   of builtin "format".
 - Adding PyObject_Format.
 - General str/unicode cleanup discovered when backporting to 2.6.
 - Removing datetimemodule.c's time_format, since it was identical
   to date_format.

The files in Objects/stringlib that implement PEP 3101 (stringdefs.h,
unicodedefs.h, formatter.h, string_format.h) are identical in trunk
and py3k.  Any changes from here on should be made to trunk, and
changes will propogate to py3k).

Include/abstract.h

@@ -611,6 +611,13 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
          */
 
 
+     PyAPI_FUNC(PyObject *) PyObject_Format(PyObject* obj,
+					    PyObject *format_spec);
+       /*
+	 Takes an arbitrary object and returns the result of
+	 calling obj.__format__(format_spec).
+       */
+
 /* Iterators */
 
      PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *);

Lib/test/test_builtin.py

@@ -541,24 +541,58 @@ class BuiltinTest(unittest.TestCase):
         self.assertRaises(TypeError, float, Foo4(42))
 
     def test_format(self):
-        class A:
-            def __init__(self, x):
-                self.x = x
-            def __format__(self, format_spec):
-                return str(self.x) + format_spec
+        # Test the basic machinery of the format() builtin.  Don't test
+        #  the specifics of the various formatters
+        self.assertEqual(format(3, ''), '3')
 
-        # class that returns a bad type from __format__
-        class B:
-            def __format__(self, format_spec):
-                return 1.0
+        # Returns some classes to use for various tests.  There's
+        #  an old-style version, and a new-style version
+        def classes_new():
+            class A(object):
+                def __init__(self, x):
+                    self.x = x
+                def __format__(self, format_spec):
+                    return str(self.x) + format_spec
+            class DerivedFromA(A):
+                pass
 
-        # class that is derived from string, used
-        #  as a format spec
-        class C(str):
-            pass
+            class Simple(object): pass
+            class DerivedFromSimple(Simple):
+                def __init__(self, x):
+                    self.x = x
+                def __format__(self, format_spec):
+                    return str(self.x) + format_spec
+            class DerivedFromSimple2(DerivedFromSimple): pass
+            return A, DerivedFromA, DerivedFromSimple, DerivedFromSimple2
+
+        # In 3.0, classes_classic has the same meaning as classes_new
+        def classes_classic():
+            class A:
+                def __init__(self, x):
+                    self.x = x
+                def __format__(self, format_spec):
+                    return str(self.x) + format_spec
+            class DerivedFromA(A):
+                pass
 
-        self.assertEqual(format(3, ''), '3')
-        self.assertEqual(format(A(3), 'spec'), '3spec')
+            class Simple: pass
+            class DerivedFromSimple(Simple):
+                def __init__(self, x):
+                    self.x = x
+                def __format__(self, format_spec):
+                    return str(self.x) + format_spec
+            class DerivedFromSimple2(DerivedFromSimple): pass
+            return A, DerivedFromA, DerivedFromSimple, DerivedFromSimple2
+
+        def class_test(A, DerivedFromA, DerivedFromSimple, DerivedFromSimple2):
+            self.assertEqual(format(A(3), 'spec'), '3spec')
+            self.assertEqual(format(DerivedFromA(4), 'spec'), '4spec')
+            self.assertEqual(format(DerivedFromSimple(5), 'abc'), '5abc')
+            self.assertEqual(format(DerivedFromSimple2(10), 'abcdef'),
+                             '10abcdef')
+
+        class_test(*classes_new())
+        class_test(*classes_classic())
 
         def empty_format_spec(value):
             # test that:
@@ -578,19 +612,28 @@ class BuiltinTest(unittest.TestCase):
         empty_format_spec(None)
 
         # TypeError because self.__format__ returns the wrong type
-        self.assertRaises(TypeError, format, B(), "")
+        class BadFormatResult:
+            def __format__(self, format_spec):
+                return 1.0
+        self.assertRaises(TypeError, format, BadFormatResult(), "")
 
-        # TypeError because format_spec is not unicode
+        # TypeError because format_spec is not unicode or str
         self.assertRaises(TypeError, format, object(), 4)
         self.assertRaises(TypeError, format, object(), object())
 
+        # tests for object.__format__ really belong elsewhere, but
+        #  there's no good place to put them
+        x = object().__format__('')
+        self.assert_(x.startswith('<object object at'))
+
         # first argument to object.__format__ must be string
         self.assertRaises(TypeError, object().__format__, 3)
         self.assertRaises(TypeError, object().__format__, object())
         self.assertRaises(TypeError, object().__format__, None)
 
         # make sure we can take a subclass of str as a format spec
-        self.assertEqual(format(0, C('10')), '         0')
+        class DerivedFromStr(str): pass
+        self.assertEqual(format(0, DerivedFromStr('10')), '         0')
 
     def test_floatasratio(self):
         for f, ratio in [

Lib/test/test_datetime.py

@@ -851,29 +851,29 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase):
 
     def test_format(self):
         dt = self.theclass(2007, 9, 10)
-        self.assertEqual(format(dt, ''), str(dt))
+        self.assertEqual(dt.__format__(''), str(dt))
 
         # check that a derived class's __str__() gets called
         class A(self.theclass):
             def __str__(self):
                 return 'A'
         a = A(2007, 9, 10)
-        self.assertEqual(format(a, ''), 'A')
+        self.assertEqual(a.__format__(''), 'A')
 
         # check that a derived class's strftime gets called
         class B(self.theclass):
             def strftime(self, format_spec):
                 return 'B'
         b = B(2007, 9, 10)
-        self.assertEqual(format(b, ''), str(dt))
+        self.assertEqual(b.__format__(''), str(dt))
 
         for fmt in ["m:%m d:%d y:%y",
                     "m:%m d:%d y:%y H:%H M:%M S:%S",
                     "%z %Z",
                     ]:
-            self.assertEqual(format(dt, fmt), dt.strftime(fmt))
-            self.assertEqual(format(a, fmt), dt.strftime(fmt))
-            self.assertEqual(format(b, fmt), 'B')
+            self.assertEqual(dt.__format__(fmt), dt.strftime(fmt))
+            self.assertEqual(a.__format__(fmt), dt.strftime(fmt))
+            self.assertEqual(b.__format__(fmt), 'B')
 
     def test_resolution_info(self):
         self.assert_(isinstance(self.theclass.min, self.theclass))
@@ -1178,31 +1178,29 @@ class TestDateTime(TestDate):
 
     def test_format(self):
         dt = self.theclass(2007, 9, 10, 4, 5, 1, 123)
-        self.assertEqual(format(dt, ''), str(dt))
+        self.assertEqual(dt.__format__(''), str(dt))
 
         # check that a derived class's __str__() gets called
         class A(self.theclass):
             def __str__(self):
                 return 'A'
         a = A(2007, 9, 10, 4, 5, 1, 123)
-        self.assertEqual(format(a, ''), 'A')
+        self.assertEqual(a.__format__(''), 'A')
 
         # check that a derived class's strftime gets called
         class B(self.theclass):
             def strftime(self, format_spec):
                 return 'B'
         b = B(2007, 9, 10, 4, 5, 1, 123)
-        self.assertEqual(format(b, ''), str(dt))
+        self.assertEqual(b.__format__(''), str(dt))
 
         for fmt in ["m:%m d:%d y:%y",
                     "m:%m d:%d y:%y H:%H M:%M S:%S",
                     "%z %Z",
                     ]:
-            self.assertEqual(format(dt, fmt), dt.strftime(fmt))
-            self.assertEqual(format(a, fmt), dt.strftime(fmt))
-            self.assertEqual(format(b, fmt), 'B')
-
-
+            self.assertEqual(dt.__format__(fmt), dt.strftime(fmt))
+            self.assertEqual(a.__format__(fmt), dt.strftime(fmt))
+            self.assertEqual(b.__format__(fmt), 'B')
 
     def test_more_ctime(self):
         # Test fields that TestDate doesn't touch.
@@ -1837,27 +1835,27 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase):
 
     def test_format(self):
         t = self.theclass(1, 2, 3, 4)
-        self.assertEqual(format(t, ''), str(t))
+        self.assertEqual(t.__format__(''), str(t))
 
         # check that a derived class's __str__() gets called
         class A(self.theclass):
             def __str__(self):
                 return 'A'
         a = A(1, 2, 3, 4)
-        self.assertEqual(format(a, ''), 'A')
+        self.assertEqual(a.__format__(''), 'A')
 
         # check that a derived class's strftime gets called
         class B(self.theclass):
             def strftime(self, format_spec):
                 return 'B'
         b = B(1, 2, 3, 4)
-        self.assertEqual(format(b, ''), str(t))
+        self.assertEqual(b.__format__(''), str(t))
 
         for fmt in ['%H %M %S',
                     ]:
-            self.assertEqual(format(t, fmt), t.strftime(fmt))
-            self.assertEqual(format(a, fmt), t.strftime(fmt))
-            self.assertEqual(format(b, fmt), 'B')
+            self.assertEqual(t.__format__(fmt), t.strftime(fmt))
+            self.assertEqual(a.__format__(fmt), t.strftime(fmt))
+            self.assertEqual(b.__format__(fmt), 'B')
 
     def test_str(self):
         self.assertEqual(str(self.theclass(1, 2, 3, 4)), "01:02:03.000004")

Modules/datetimemodule.c

@@ -3210,21 +3210,6 @@ time_strftime(PyDateTime_Time *self, PyObject *args, PyObject *kw)
 	return result;
 }
 
-static PyObject *
-time_format(PyDateTime_Time *self, PyObject *args)
-{
-	PyObject *format;
-
-	if (!PyArg_ParseTuple(args, "U:__format__", &format))
-		return NULL;
-
-	/* if the format is zero length, return str(self) */
-	if (PyUnicode_GetSize(format) == 0)
-                return PyObject_Str((PyObject *)self);
-
-        return PyObject_CallMethod((PyObject *)self, "strftime", "O", format);
-}
-
 /*
  * Miscellaneous methods.
  */
@@ -3412,7 +3397,7 @@ static PyMethodDef time_methods[] = {
 	{"strftime",   	(PyCFunction)time_strftime,	METH_VARARGS | METH_KEYWORDS,
 	 PyDoc_STR("format -> strftime() style string.")},
 
-	{"__format__", 	(PyCFunction)time_format,	METH_VARARGS,
+	{"__format__", 	(PyCFunction)date_format,	METH_VARARGS,
 	 PyDoc_STR("Formats self with strftime.")},
 
 	{"utcoffset",	(PyCFunction)time_utcoffset,	METH_NOARGS,

Objects/abstract.c

@@ -704,6 +704,57 @@ PyBuffer_FillInfo(Py_buffer *view, void *buf, Py_ssize_t len,
 	return 0;
 }
 
+PyObject *
+PyObject_Format(PyObject *obj, PyObject *format_spec)
+{
+    static PyObject * str__format__ = NULL;
+    PyObject *meth;
+    PyObject *empty = NULL;
+    PyObject *result = NULL;
+
+    /* Initialize cached value */
+    if (str__format__ == NULL) {
+        /* Initialize static variable needed by _PyType_Lookup */
+        str__format__ = PyUnicode_FromString("__format__");
+        if (str__format__ == NULL)
+            goto done;
+    }
+
+    /* If no format_spec is provided, use an empty string */
+    if (format_spec == NULL) {
+        empty = PyUnicode_FromUnicode(NULL, 0);
+        format_spec = empty;
+    }
+
+    /* Make sure the type is initialized.  float gets initialized late */
+    if (Py_TYPE(obj)->tp_dict == NULL)
+        if (PyType_Ready(Py_TYPE(obj)) < 0)
+            goto done;
+
+    /* Find the (unbound!) __format__ method (a borrowed reference) */
+    meth = _PyType_Lookup(Py_TYPE(obj), str__format__);
+    if (meth == NULL) {
+        PyErr_Format(PyExc_TypeError,
+                "Type %.100s doesn't define __format__",
+                Py_TYPE(obj)->tp_name);
+            goto done;
+    }
+
+    /* And call it, binding it to the value */
+    result = PyObject_CallFunctionObjArgs(meth, obj, format_spec, NULL);
+
+    if (result && !PyUnicode_Check(result)) {
+        PyErr_SetString(PyExc_TypeError,
+            "__format__ method did not return string");
+        Py_DECREF(result);
+        result = NULL;
+        goto done;
+    }
+
+done:
+    Py_XDECREF(empty);
+    return result;
+}
 /* Operations on numbers */
 
 int

Objects/stringlib/string_format.h

@@ -6,6 +6,11 @@
 */
 
 
+/* Defines for Python 2.6 compatability */
+#if PY_VERSION_HEX < 0x03000000
+#define PyLong_FromSsize_t _PyLong_FromSsize_t
+#endif
+
 /* Defines for more efficiently reallocating the string buffer */
 #define INITIAL_SIZE_INCREMENT 100
 #define SIZE_MULTIPLIER 2
@@ -470,66 +475,6 @@ error:
     field object and field specification string generated by
     get_field_and_spec, and renders the field into the output string.
 
-    format() does the actual calling of the objects __format__ method.
-*/
-
-
-/* returns fieldobj.__format__(format_spec) */
-static PyObject *
-format(PyObject *fieldobj, SubString *format_spec)
-{
-    static PyObject *format_str = NULL;
-    PyObject *meth;
-    PyObject *spec = NULL;
-    PyObject *result = NULL;
-
-    /* Initialize cached value */
-    if (format_str == NULL) {
-        /* Initialize static variable needed by _PyType_Lookup */
-        format_str = PyUnicode_FromString("__format__");
-        if (format_str == NULL)
-            return NULL;
-    }
-
-    /* Make sure the type is initialized.  float gets initialized late */
-    if (Py_TYPE(fieldobj)->tp_dict == NULL)
-        if (PyType_Ready(Py_TYPE(fieldobj)) < 0)
-            return NULL;
-
-    /* we need to create an object out of the pointers we have */
-    spec = SubString_new_object_or_empty(format_spec);
-    if (spec == NULL)
-        goto done;
-
-    /* Find the (unbound!) __format__ method (a borrowed reference) */
-    meth = _PyType_Lookup(Py_TYPE(fieldobj), format_str);
-    if (meth == NULL) {
-        PyErr_Format(PyExc_TypeError,
-                     "Type %.100s doesn't define __format__",
-                     Py_TYPE(fieldobj)->tp_name);
-        goto done;
-    }
-
-    /* And call it, binding it to the value */
-    result = PyObject_CallFunctionObjArgs(meth, fieldobj, spec, NULL);
-    if (result == NULL)
-        goto done;
-
-    if (!STRINGLIB_CHECK(result)) {
-        PyErr_SetString(PyExc_TypeError,
-                        "__format__ method did not return "
-                        STRINGLIB_TYPE_NAME);
-        Py_DECREF(result);
-        result = NULL;
-        goto done;
-    }
-
-done:
-    Py_XDECREF(spec);
-    return result;
-}
-
-/*
     render_field calls fieldobj.__format__(format_spec) method, and
     appends to the output.
 */
@@ -537,14 +482,21 @@ static int
 render_field(PyObject *fieldobj, SubString *format_spec, OutputString *output)
 {
     int ok = 0;
-    PyObject *result = format(fieldobj, format_spec);
+    PyObject *result = NULL;
 
+    /* we need to create an object out of the pointers we have */
+    PyObject *format_spec_object = SubString_new_object_or_empty(format_spec);
+    if (format_spec_object == NULL)
+        goto done;
+
+    result = PyObject_Format(fieldobj, format_spec_object);
     if (result == NULL)
         goto done;
 
     ok = output_data(output,
                      STRINGLIB_STR(result), STRINGLIB_LEN(result));
 done:
+    Py_DECREF(format_spec_object);
     Py_XDECREF(result);
     return ok;
 }
@@ -770,7 +722,7 @@ do_conversion(PyObject *obj, STRINGLIB_CHAR conversion)
     case 'r':
         return PyObject_Repr(obj);
     case 's':
-        return PyObject_Str(obj);
+        return STRINGLIB_TOSTR(obj);
     default:
         PyErr_Format(PyExc_ValueError,
                      "Unknown converion specifier %c",
@@ -845,7 +797,7 @@ done:
 }
 
 /*
-    do_markup is the top-level loop for the format() function.  It
+    do_markup is the top-level loop for the format() method.  It
     searches through the format string for escapes to markup codes, and
     calls other functions to move non-markup text to the output,
     and to perform the markup to the output.
@@ -958,7 +910,7 @@ do_string_format(PyObject *self, PyObject *args, PyObject *kwargs)
 typedef struct {
     PyObject_HEAD
 
-    PyUnicodeObject *str;
+    STRINGLIB_OBJECT *str;
 
     MarkupIterator it_markup;
 } formatteriterobject;
@@ -984,7 +936,7 @@ formatteriter_next(formatteriterobject *it)
     SubString literal;
     SubString field_name;
     SubString format_spec;
-    Py_UNICODE conversion;
+    STRINGLIB_CHAR conversion;
     int format_spec_needs_expanding;
     int result = MarkupIterator_next(&it->it_markup, &literal, &field_name,
                                      &format_spec, &conversion,
@@ -1028,7 +980,7 @@ formatteriter_next(formatteriterobject *it)
             Py_INCREF(conversion_str);
         }
         else
-            conversion_str = PyUnicode_FromUnicode(&conversion, 1);
+	    conversion_str = STRINGLIB_NEW(&conversion, 1);
         if (conversion_str == NULL)
             goto done;
 
@@ -1047,7 +999,7 @@ static PyMethodDef formatteriter_methods[] = {
     {NULL,		NULL}		/* sentinel */
 };
 
-PyTypeObject PyFormatterIter_Type = {
+static PyTypeObject PyFormatterIter_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "formatteriterator",		/* tp_name */
     sizeof(formatteriterobject),	/* tp_basicsize */
@@ -1085,7 +1037,7 @@ PyTypeObject PyFormatterIter_Type = {
    describing the parsed elements.  It's a wrapper around
    stringlib/string_format.h's MarkupIterator */
 static PyObject *
-formatter_parser(PyUnicodeObject *self)
+formatter_parser(STRINGLIB_OBJECT *self)
 {
     formatteriterobject *it;
 
@@ -1099,8 +1051,8 @@ formatter_parser(PyUnicodeObject *self)
 
     /* initialize the contained MarkupIterator */
     MarkupIterator_init(&it->it_markup,
-                        PyUnicode_AS_UNICODE(self),
-                        PyUnicode_GET_SIZE(self));
+                        STRINGLIB_STR(self),
+                        STRINGLIB_LEN(self));
 
     return (PyObject *)it;
 }
@@ -1118,7 +1070,7 @@ formatter_parser(PyUnicodeObject *self)
 typedef struct {
     PyObject_HEAD
 
-    PyUnicodeObject *str;
+    STRINGLIB_OBJECT *str;
 
     FieldNameIterator it_field;
 } fieldnameiterobject;
@@ -1220,7 +1172,7 @@ static PyTypeObject PyFieldNameIter_Type = {
    field_name_split.  The iterator it returns is a
    FieldNameIterator */
 static PyObject *
-formatter_field_name_split(PyUnicodeObject *self)
+formatter_field_name_split(STRINGLIB_OBJECT *self)
 {
     SubString first;
     Py_ssize_t first_idx;

Objects/stringlib/stringdefs.h

@@ -6,12 +6,15 @@
    compiled as unicode. */
 #define STRINGLIB_IS_UNICODE     0
 
+#define STRINGLIB_OBJECT         PyStringObject
 #define STRINGLIB_CHAR           char
 #define STRINGLIB_TYPE_NAME      "string"
 #define STRINGLIB_PARSE_CODE     "S"
-#define STRINGLIB_EMPTY          string_empty
+#define STRINGLIB_EMPTY          nullstring
 #define STRINGLIB_ISDECIMAL(x)   ((x >= '0') && (x <= '9'))
 #define STRINGLIB_TODECIMAL(x)   (STRINGLIB_ISDECIMAL(x) ? (x - '0') : -1)
+#define STRINGLIB_TOUPPER        toupper
+#define STRINGLIB_TOLOWER        tolower
 #define STRINGLIB_FILL           memset
 #define STRINGLIB_STR            PyString_AS_STRING
 #define STRINGLIB_LEN            PyString_GET_SIZE

Objects/stringlib/unicodedefs.h

@@ -6,6 +6,7 @@
    compiled as unicode. */
 #define STRINGLIB_IS_UNICODE     1
 
+#define STRINGLIB_OBJECT         PyUnicodeObject
 #define STRINGLIB_CHAR           Py_UNICODE
 #define STRINGLIB_TYPE_NAME      "unicode"
 #define STRINGLIB_PARSE_CODE     "U"
@@ -20,7 +21,12 @@
 #define STRINGLIB_NEW            PyUnicode_FromUnicode
 #define STRINGLIB_RESIZE         PyUnicode_Resize
 #define STRINGLIB_CHECK          PyUnicode_Check
+
+#if PY_VERSION_HEX < 0x03000000
+#define STRINGLIB_TOSTR          PyObject_Unicode
+#else
 #define STRINGLIB_TOSTR          PyObject_Str
+#endif
 
 #define STRINGLIB_WANT_CONTAINS_OBJ 1
 

Python/bltinmodule.c

@@ -304,58 +304,13 @@ If the predicate is None, 'lambda x: bool(x)' is assumed.\n\
 static PyObject *
 builtin_format(PyObject *self, PyObject *args)
 {
-    static PyObject * format_str = NULL;
     PyObject *value;
-    PyObject *spec = NULL;
-    PyObject *meth;
-    PyObject *empty = NULL;
-    PyObject *result = NULL;
-
-    /* Initialize cached value */
-    if (format_str == NULL) {
-        /* Initialize static variable needed by _PyType_Lookup */
-        format_str = PyUnicode_FromString("__format__");
-        if (format_str == NULL)
-            goto done;
-    }
-
-    if (!PyArg_ParseTuple(args, "O|U:format", &value, &spec))
-        goto done;
-
-    /* initialize the default value */
-    if (spec == NULL) {
-        empty = PyUnicode_FromUnicode(NULL, 0);
-        spec = empty;
-    }
-
-    /* Make sure the type is initialized.  float gets initialized late */
-    if (Py_TYPE(value)->tp_dict == NULL)
-        if (PyType_Ready(Py_TYPE(value)) < 0)
-            goto done;
-
-    /* Find the (unbound!) __format__ method (a borrowed reference) */
-    meth = _PyType_Lookup(Py_TYPE(value), format_str);
-    if (meth == NULL) {
-        PyErr_Format(PyExc_TypeError,
-                "Type %.100s doesn't define __format__",
-                Py_TYPE(value)->tp_name);
-            goto done;
-    }
-
-    /* And call it, binding it to the value */
-    result = PyObject_CallFunctionObjArgs(meth, value, spec, NULL);
-
-    if (result && !PyUnicode_Check(result)) {
-        PyErr_SetString(PyExc_TypeError,
-            "__format__ method did not return string");
-        Py_DECREF(result);
-        result = NULL;
-        goto done;
-    }
-
-done:
-    Py_XDECREF(empty);
-    return result;
+    PyObject *format_spec = NULL;
+
+    if (!PyArg_ParseTuple(args, "O|U:format", &value, &format_spec))
+        return NULL;
+
+    return PyObject_Format(value, format_spec);
 }
 
 PyDoc_STRVAR(format_doc,