Issue #11145: Fixed miscellaneous issues with C-style formatting of types

with custom __oct__ and __hex__.

Lib/test/test_format.py

@@ -300,6 +300,44 @@ class FormatTest(unittest.TestCase):
             else:
                 raise TestFailed, '"%*d"%(maxsize, -127) should fail'
 
+    def test_invalid_special_methods(self):
+        tests = []
+        for f in 'sriduoxXfge':
+            tests.append(('%' + f, 1, TypeError))
+            tests.append(('%#' + f, 1, TypeError))
+        for r in ['', '-', 'L', '-L']:
+            for f in 'iduoxX':
+                tests.append(('%' + f, r, ValueError))
+                tests.append(('%#' + f, r, ValueError))
+        tests.append(('%o', 'abc', ValueError))
+        for r in ('abc', '0abc', '0x', '0xL'):
+            for f in 'xX':
+                tests.append(('%' + f, r, ValueError))
+        for r in ('0x', '0xL'):
+            for f in 'xX':
+                tests.append(('%#' + f, r, ValueError))
+
+        class X(long):
+            def __repr__(self):
+                return result
+            def __str__(self):
+                return result
+            def __oct__(self):
+                return result
+            def __hex__(self):
+                return result
+            def __float__(self):
+                return result
+        for fmt, result, exc in tests:
+            try:
+                fmt % X()
+            except exc:
+                pass
+            else:
+                self.fail('%s not raised for %r format of %r' %
+                          (exc.__name__, fmt, result))
+
+
 def test_main():
     test_support.run_unittest(FormatTest)
 

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 2.7.13?
 Core and Builtins
 -----------------
 
+- Issue #11145: Fixed miscellaneous issues with C-style formatting of types
+  with custom __oct__ and __hex__.
+
 - Issue #24469: Fixed memory leak caused by int subclasses without overridden
   tp_free (e.g. C-inherited Cython classes).
 

Objects/stringobject.c

@@ -4006,26 +4006,30 @@ PyObject*
 _PyString_FormatLong(PyObject *val, int flags, int prec, int type,
                      char **pbuf, int *plen)
 {
-    PyObject *result = NULL;
+    PyObject *result = NULL, *r1;
+    const char *s;
     char *buf;
     Py_ssize_t i;
     int sign;           /* 1 if '-', else 0 */
     int len;            /* number of characters */
     Py_ssize_t llen;
-    int numdigits;      /* len == numnondigits + numdigits */
-    int numnondigits = 0;
+    int numdigits;      /* len == numnondigits + skipped + numdigits */
+    int numnondigits, skipped, filled;
+    const char *method;
 
     switch (type) {
     case 'd':
     case 'u':
+        method = "str";
         result = Py_TYPE(val)->tp_str(val);
         break;
     case 'o':
+        method = "oct";
         result = Py_TYPE(val)->tp_as_number->nb_oct(val);
         break;
     case 'x':
     case 'X':
-        numnondigits = 2;
+        method = "hex";
         result = Py_TYPE(val)->tp_as_number->nb_hex(val);
         break;
     default:
@@ -4034,97 +4038,109 @@ _PyString_FormatLong(PyObject *val, int flags, int prec, int type,
     if (!result)
         return NULL;
 
-    buf = PyString_AsString(result);
-    if (!buf) {
+    if (PyString_AsStringAndSize(result, (char **)&s, &llen) < 0) {
         Py_DECREF(result);
         return NULL;
     }
-
-    /* To modify the string in-place, there can only be one reference. */
-    if (Py_REFCNT(result) != 1) {
-        PyErr_BadInternalCall();
-        return NULL;
-    }
-    llen = PyString_Size(result);
     if (llen > INT_MAX) {
         PyErr_SetString(PyExc_ValueError, "string too large in _PyString_FormatLong");
+        Py_DECREF(result);
         return NULL;
     }
     len = (int)llen;
-    if (buf[len-1] == 'L') {
+    if (len > 0 && s[len-1] == 'L') {
         --len;
-        buf[len] = '\0';
-    }
-    sign = buf[0] == '-';
-    numnondigits += sign;
-    numdigits = len - numnondigits;
-    assert(numdigits > 0);
-
-    /* Get rid of base marker unless F_ALT */
-    if ((flags & F_ALT) == 0) {
-        /* Need to skip 0x, 0X or 0. */
-        int skipped = 0;
-        switch (type) {
-        case 'o':
-            assert(buf[sign] == '0');
-            /* If 0 is only digit, leave it alone. */
-            if (numdigits > 1) {
-                skipped = 1;
-                --numdigits;
-            }
-            break;
-        case 'x':
-        case 'X':
-            assert(buf[sign] == '0');
-            assert(buf[sign + 1] == 'x');
+        if (len == 0)
+            goto error;
+    }
+    sign = s[0] == '-';
+    numnondigits = sign;
+
+    /* Need to skip 0x, 0X or 0. */
+    skipped = 0;
+    switch (type) {
+    case 'o':
+        if (s[sign] != '0')
+            goto error;
+        /* If 0 is only digit, leave it alone. */
+        if ((flags & F_ALT) == 0 && len - sign > 1)
+            skipped = 1;
+        break;
+    case 'x':
+    case 'X':
+        if (s[sign] != '0' || (s[sign + 1] != 'x' && s[sign + 1] != 'X'))
+            goto error;
+        if ((flags & F_ALT) == 0)
             skipped = 2;
-            numnondigits -= 2;
-            break;
-        }
-        if (skipped) {
-            buf += skipped;
-            len -= skipped;
-            if (sign)
-                buf[0] = '-';
-        }
-        assert(len == numnondigits + numdigits);
-        assert(numdigits > 0);
+        else
+            numnondigits += 2;
+        break;
     }
+    numdigits = len - numnondigits - skipped;
+    if (numdigits <= 0)
+        goto error;
+
+    filled = prec - numdigits;
+    if (filled < 0)
+        filled = 0;
+    len = numnondigits + filled + numdigits;
 
-    /* Fill with leading zeroes to meet minimum width. */
-    if (prec > numdigits) {
-        PyObject *r1 = PyString_FromStringAndSize(NULL,
-                                numnondigits + prec);
-        char *b1;
-        if (!r1) {
-            Py_DECREF(result);
+    /* To modify the string in-place, there can only be one reference. */
+    if (skipped >= filled &&
+        PyString_CheckExact(result) &&
+        Py_REFCNT(result) == 1 &&
+        !PyString_CHECK_INTERNED(result))
+    {
+        r1 = NULL;
+        buf = (char *)s + skipped - filled;
+    }
+    else {
+        r1 = result;
+        result = PyString_FromStringAndSize(NULL, len);
+        if (!result) {
+            Py_DECREF(r1);
             return NULL;
         }
-        b1 = PyString_AS_STRING(r1);
-        for (i = 0; i < numnondigits; ++i)
-            *b1++ = *buf++;
-        for (i = 0; i < prec - numdigits; i++)
-            *b1++ = '0';
-        for (i = 0; i < numdigits; i++)
-            *b1++ = *buf++;
-        *b1 = '\0';
-        Py_DECREF(result);
-        result = r1;
         buf = PyString_AS_STRING(result);
-        len = numnondigits + prec;
     }
 
+    for (i = numnondigits; --i >= 0;)
+        buf[i] = s[i];
+    buf += numnondigits;
+    s += numnondigits + skipped;
+    for (i = 0; i < filled; i++)
+        *buf++ = '0';
+    if (r1 == NULL) {
+        assert(buf == s);
+        buf += numdigits;
+    }
+    else {
+        for (i = 0; i < numdigits; i++)
+            *buf++ = *s++;
+    }
+    *buf = '\0';
+    buf -= len;
+    Py_XDECREF(r1);
+
     /* Fix up case for hex conversions. */
     if (type == 'X') {
         /* Need to convert all lower case letters to upper case.
            and need to convert 0x to 0X (and -0x to -0X). */
-        for (i = 0; i < len; i++)
-            if (buf[i] >= 'a' && buf[i] <= 'x')
+        for (i = 0; i < len; i++) {
+            if (buf[i] >= 'a' && buf[i] <= 'z')
                 buf[i] -= 'a'-'A';
+        }
     }
     *pbuf = buf;
     *plen = len;
     return result;
+
+error:
+    PyErr_Format(PyExc_ValueError,
+                 "%%%c format: invalid result of __%s__ (type=%.200s)",
+                 type, method, Py_TYPE(val)->tp_name);
+    Py_DECREF(result);
+    return NULL;
 }
 
 Py_LOCAL_INLINE(int)