The other half of Issue #1580: use short float repr where possible.

Addresses the float -> string conversion, using David Gay's code which
was added in Mark Dickinson's checkin r71663.

Also addresses these, which are intertwined with the short repr
changes:

- Issue #5772: format(1e100, '<') produces '1e+100', not '1.0e+100'
- Issue #5515: 'n' formatting with commas no longer works poorly
    with leading zeros.
- PEP 378 Format Specifier for Thousands Separator: implemented
    for floats.

Include/bytesobject.h

@@ -91,24 +91,22 @@ PyAPI_FUNC(int) PyBytes_AsStringAndSize(
    into the string pointed to by buffer.  For the argument descriptions,
    see Objects/stringlib/localeutil.h */
 
-PyAPI_FUNC(int) _PyBytes_InsertThousandsGroupingLocale(char *buffer,
-						  Py_ssize_t n_buffer,
-						  Py_ssize_t n_digits,
-						  Py_ssize_t buf_size,
-						  Py_ssize_t *count,
-						  int append_zero_char);
+PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGroupingLocale(char *buffer,
+                                                   Py_ssize_t n_buffer,
+                                                   char *digits,
+                                                   Py_ssize_t n_digits,
+                                                   Py_ssize_t min_width);
 
 /* Using explicit passed-in values, insert the thousands grouping
    into the string pointed to by buffer.  For the argument descriptions,
    see Objects/stringlib/localeutil.h */
-PyAPI_FUNC(int) _PyBytes_InsertThousandsGrouping(char *buffer,
-						 Py_ssize_t n_buffer,
-						 Py_ssize_t n_digits,
-						 Py_ssize_t buf_size,
-						 Py_ssize_t *count,
-                                                 int append_zero_char,
-                                                 const char *grouping,
-                                                 const char *thousands_sep);
+PyAPI_FUNC(Py_ssize_t) _PyBytes_InsertThousandsGrouping(char *buffer,
+                                                   Py_ssize_t n_buffer,
+                                                   char *digits,
+                                                   Py_ssize_t n_digits,
+                                                   Py_ssize_t min_width,
+                                                   const char *grouping,
+                                                   const char *thousands_sep);
 
 /* Flags used by string formatting */
 #define F_LJUST (1<<0)

Include/pystrtod.h

@@ -10,6 +10,25 @@ PyAPI_FUNC(double) PyOS_ascii_strtod(const char *str, char **ptr);
 PyAPI_FUNC(double) PyOS_ascii_atof(const char *str);
 PyAPI_FUNC(char *) PyOS_ascii_formatd(char *buffer, size_t buf_len,  const char *format, double d);
 
+/* The caller is responsible for calling PyMem_Free to free the buffer
+   that's is returned. */
+PyAPI_FUNC(char *) PyOS_double_to_string(double val,
+                                         char format_code,
+                                         int precision,
+                                         int flags,
+                                         int *type);
+
+
+/* PyOS_double_to_string's "flags" parameter can be set to 0 or more of: */
+#define Py_DTSF_SIGN      0x01 /* always add the sign */
+#define Py_DTSF_ADD_DOT_0 0x02 /* if the result is an integer add ".0" */
+#define Py_DTSF_ALT       0x04 /* "alternate" formatting. it's format_code
+                                  specific */
+
+/* PyOS_double_to_string's "type", if non-NULL, will be set to one of: */
+#define Py_DTST_FINITE 0
+#define Py_DTST_INFINITE 1
+#define Py_DTST_NAN 2
 
 #ifdef __cplusplus
 }

Include/unicodeobject.h

@@ -1482,24 +1482,22 @@ PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
    into the string pointed to by buffer.  For the argument descriptions,
    see Objects/stringlib/localeutil.h */
 
-PyAPI_FUNC(int) _PyUnicode_InsertThousandsGroupingLocale(Py_UNICODE *buffer,
-						  Py_ssize_t n_buffer,
-						  Py_ssize_t n_digits,
-						  Py_ssize_t buf_size,
-						  Py_ssize_t *count,
-						  int append_zero_char);
+PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGroupingLocale(Py_UNICODE *buffer,
+                                                   Py_ssize_t n_buffer,
+                                                   Py_UNICODE *digits,
+                                                   Py_ssize_t n_digits,
+                                                   Py_ssize_t min_width);
 
 /* Using explicit passed-in values, insert the thousands grouping
    into the string pointed to by buffer.  For the argument descriptions,
    see Objects/stringlib/localeutil.h */
-PyAPI_FUNC(int) _PyUnicode_InsertThousandsGrouping(Py_UNICODE *buffer,
-						 Py_ssize_t n_buffer,
-						 Py_ssize_t n_digits,
-						 Py_ssize_t buf_size,
-						 Py_ssize_t *count,
-                                                 int append_zero_char,
-                                                 const char *grouping,
-                                                 const char *thousands_sep);
+PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(Py_UNICODE *buffer,
+                                                   Py_ssize_t n_buffer,
+                                                   Py_UNICODE *digits,
+                                                   Py_ssize_t n_digits,
+                                                   Py_ssize_t min_width,
+                                                   const char *grouping,
+                                                   const char *thousands_sep);
 /* === Characters Type APIs =============================================== */
 
 /* Helper array used by Py_UNICODE_ISSPACE(). */

Lib/test/test_float.py

 
 import unittest, struct
 import os
+import sys
 from test import support
 import math
 from math import isinf, isnan, copysign, ldexp
@@ -10,6 +11,10 @@ import random, fractions
 INF = float("inf")
 NAN = float("nan")
 
+#locate file with float format test values
+test_dir = os.path.dirname(__file__) or os.curdir
+format_testfile = os.path.join(test_dir, 'formatfloat_testcases.txt')
+
 class GeneralFloatCases(unittest.TestCase):
 
     def test_float(self):
@@ -24,6 +29,10 @@ class GeneralFloatCases(unittest.TestCase):
         self.assertRaises(ValueError, float, "+-3.14")
         self.assertRaises(ValueError, float, "-+3.14")
         self.assertRaises(ValueError, float, "--3.14")
+        self.assertRaises(ValueError, float, ".nan")
+        self.assertRaises(ValueError, float, "+.inf")
+        self.assertRaises(ValueError, float, ".")
+        self.assertRaises(ValueError, float, "-.")
         self.assertEqual(float(b"  \u0663.\u0661\u0664  ".decode('raw-unicode-escape')), 3.14)
 
     @support.run_with_locale('LC_NUMERIC', 'fr_FR', 'de_DE')
@@ -316,6 +325,73 @@ class ReprTestCase(unittest.TestCase):
             self.assertEqual(v, eval(repr(v)))
         floats_file.close()
 
+class FormatTestCase(unittest.TestCase):
+    @unittest.skipUnless(float.__getformat__("double").startswith("IEEE"),
+                         "test requires IEEE 754 doubles")
+    def test_format_testfile(self):
+        for line in open(format_testfile):
+            if line.startswith('--'):
+                continue
+            line = line.strip()
+            if not line:
+                continue
+
+            lhs, rhs = map(str.strip, line.split('->'))
+            fmt, arg = lhs.split()
+            self.assertEqual(fmt % float(arg), rhs)
+            self.assertEqual(fmt % -float(arg), '-' + rhs)
+
+    @unittest.skipUnless(getattr(sys, 'float_repr_style', '') == 'short',
+                         "applies only when using short float repr style")
+    def test_short_repr(self):
+        # test short float repr introduced in Python 3.1.  One aspect
+        # of this repr is that we get some degree of str -> float ->
+        # str roundtripping.  In particular, for any numeric string
+        # containing 15 or fewer significant digits, those exact same
+        # digits (modulo trailing zeros) should appear in the output.
+        # No more repr(0.03) -> "0.029999999999999999"!
+
+        test_strings = [
+            # output always includes *either* a decimal point and at
+            # least one digit after that point, or an exponent.
+            '0.0',
+            '1.0',
+            '0.01',
+            '0.02',
+            '0.03',
+            '0.04',
+            '0.05',
+            '1.23456789',
+            '10.0',
+            '100.0',
+            # values >= 1e16 get an exponent...
+            '1000000000000000.0',
+            '9999999999999990.0',
+            '1e+16',
+            '1e+17',
+            # ... and so do values < 1e-4
+            '0.001',
+            '0.001001',
+            '0.00010000000000001',
+            '0.0001',
+            '9.999999999999e-05',
+            '1e-05',
+            # values designed to provoke failure if the FPU rounding
+            # precision isn't set correctly
+            '8.72293771110361e+25',
+            '7.47005307342313e+26',
+            '2.86438000439698e+28',
+            '8.89142905246179e+28',
+            '3.08578087079232e+35',
+            ]
+
+        for s in test_strings:
+            negs = '-'+s
+            self.assertEqual(s, repr(float(s)))
+            self.assertEqual(negs, repr(float(negs)))
+
+
+
 # Beginning with Python 2.6 float has cross platform compatible
 # ways to create and represent inf and nan
 class InfNanTest(unittest.TestCase):

Lib/test/test_format.py

@@ -220,6 +220,11 @@ class FormatTest(unittest.TestCase):
         testformat("%a", "\u0378", "'\\u0378'")  # non printable
         testformat("%r", "\u0374", "'\u0374'")   # printable
         testformat("%a", "\u0374", "'\\u0374'")  # printable
+
+        # alternate float formatting
+        testformat('%g', 1.1, '1.1')
+        testformat('%#g', 1.1, '1.10000')
+
         # Test exception for unknown format characters
         if verbose:
             print('Testing exceptions')

Lib/test/test_types.py

@@ -113,6 +113,9 @@ class TypesTests(unittest.TestCase):
         self.assertEqual(1.5e-101.__format__('e'), '1.500000e-101')
         self.assertEqual('%e' % 1.5e-101, '1.500000e-101')
 
+        self.assertEqual('%g' % 1.0, '1')
+        self.assertEqual('%#g' % 1.0, '1.00000')
+
     def test_normal_integers(self):
         # Ensure the first 256 integers are shared
         a = 256
@@ -358,6 +361,8 @@ class TypesTests(unittest.TestCase):
         self.assertRaises(TypeError, 3 .__format__, 0)
         # can't have ',' with 'n'
         self.assertRaises(ValueError, 3 .__format__, ",n")
+        # can't have ',' with 'c'
+        self.assertRaises(ValueError, 3 .__format__, ",c")
 
         # ensure that only int and float type specifiers work
         for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] +
@@ -547,10 +552,34 @@ class TypesTests(unittest.TestCase):
         # a totaly empty format specifier means something else.
         # So, just use a sign flag
         test(1e200, '+g', '+1e+200')
-        test(1e200, '+', '+1.0e+200')
+        test(1e200, '+', '+1e+200')
+
         test(1.1e200, '+g', '+1.1e+200')
         test(1.1e200, '+', '+1.1e+200')
 
+        # 0 padding
+        test(1234., '010f', '1234.000000')
+        test(1234., '011f', '1234.000000')
+        test(1234., '012f', '01234.000000')
+        test(-1234., '011f', '-1234.000000')
+        test(-1234., '012f', '-1234.000000')
+        test(-1234., '013f', '-01234.000000')
+        test(-1234.12341234, '013f', '-01234.123412')
+        test(-123456.12341234, '011.2f', '-0123456.12')
+
+        # 0 padding with commas
+        test(1234., '011,f', '1,234.000000')
+        test(1234., '012,f', '1,234.000000')
+        test(1234., '013,f', '01,234.000000')
+        test(-1234., '012,f', '-1,234.000000')
+        test(-1234., '013,f', '-1,234.000000')
+        test(-1234., '014,f', '-01,234.000000')
+        test(-12345., '015,f', '-012,345.000000')
+        test(-123456., '016,f', '-0,123,456.000000')
+        test(-123456., '017,f', '-0,123,456.000000')
+        test(-123456.12341234, '017,f', '-0,123,456.123412')
+        test(-123456.12341234, '013,.2f', '-0,123,456.12')
+
         # % formatting
         test(-1.0, '%', '-100.000000%')
 
@@ -575,6 +604,24 @@ class TypesTests(unittest.TestCase):
         self.assertRaises(ValueError, format, 0.0, '#')
         self.assertRaises(ValueError, format, 0.0, '#20f')
 
+    def test_format_spec_errors(self):
+        # int, float, and string all share the same format spec
+        # mini-language parser.
+
+        # Check that we can't ask for too many digits. This is
+        # probably a CPython specific test. It tries to put the width
+        # into a C long.
+        self.assertRaises(ValueError, format, 0, '1'*10000 + 'd')
+
+        # Similar with the precision.
+        self.assertRaises(ValueError, format, 0, '.' + '1'*10000 + 'd')
+
+        # And may as well test both.
+        self.assertRaises(ValueError, format, 0, '1'*1000 + '.' + '1'*10000 + 'd')
+
+        # Make sure commas aren't allowed with various type codes
+        for code in 'xXobns':
+            self.assertRaises(ValueError, format, 0, ',' + code)
 
 def test_main():
     run_unittest(TypesTests)

Misc/NEWS

@@ -12,6 +12,15 @@ What's New in Python 3.1 beta 1?
 Core and Builtins
 -----------------
 
+- Issue #5772: format(1e100, '<') produces '1e+100', not '1.0e+100'.
+
+- Issue #5515: str.format() presentation type 'n' with commas no
+  longer works poorly with leading zeros when formatting ints and
+  floats.
+
+- Implement PEP 378, Format Specifier for Thousands Separator, for
+  floats.
+
 - The repr function switches to exponential notation at 1e16, not 1e17
   as it did before.  This change applies to both 'short' and legacy
   float repr styles.  For the new repr style, it avoids misleading

Modules/_pickle.c

@@ -1016,16 +1016,31 @@ save_float(PicklerObject *self, PyObject *obj)
             return -1;
         if (pickler_write(self, pdata, 9) < 0)
             return -1;
-    }
+   } 
     else {
-        char pdata[250];
-        pdata[0] = FLOAT;
-        PyOS_ascii_formatd(pdata + 1, sizeof(pdata) - 2, "%.17g", x);
-        /* Extend the formatted string with a newline character */
-        strcat(pdata, "\n");
+        int result = -1;
+        char *buf = NULL;
+        char op = FLOAT;
 
-        if (pickler_write(self, pdata, strlen(pdata)) < 0)
-            return -1;
+        if (pickler_write(self, &op, 1) < 0)
+            goto done;
+
+        buf = PyOS_double_to_string(x, 'r', 0, 0, NULL);
+        if (!buf) {
+            PyErr_NoMemory();
+            goto done;
+        }
+
+        if (pickler_write(self, buf, strlen(buf)) < 0)
+            goto done;
+
+        if (pickler_write(self, "\n", 1) < 0)
+            goto done;
+
+        result = 0;
+done:
+        PyMem_Free(buf);
+        return result;
     }
 
     return 0;

Objects/bytesobject.c

@@ -562,6 +562,7 @@ PyBytes_AsStringAndSize(register PyObject *obj,
 /* -------------------------------------------------------------------- */
 /* Methods */
 
+#include "stringlib/stringdefs.h"
 #define STRINGLIB_CHAR char
 
 #define STRINGLIB_CMP memcmp

Objects/complexobject.c

@@ -14,22 +14,6 @@
 
 #ifndef WITHOUT_COMPLEX
 
-/* Precisions used by repr() and str(), respectively.
-
-   The repr() precision (17 significant decimal digits) is the minimal number
-   that is guaranteed to have enough precision so that if the number is read
-   back in the exact same binary value is recreated.  This is true for IEEE
-   floating point by design, and also happens to work for all other modern
-   hardware.
-
-   The str() precision is chosen so that in most cases, the rounding noise
-   created by various operations is suppressed, while giving plenty of
-   precision for practical use.
-*/
-
-#define PREC_REPR	17
-#define PREC_STR	12
-
 /* elementary operations on complex numbers */
 
 static Py_complex c_1 = {1., 0.};
@@ -345,71 +329,114 @@ complex_dealloc(PyObject *op)
 }
 
 
-static void
-complex_to_buf(char *buf, int bufsz, PyComplexObject *v, int precision)
+static PyObject *
+complex_format(PyComplexObject *v, char format_code)
 {
-	char format[32];
-	if (v->cval.real == 0.) {
-		if (!Py_IS_FINITE(v->cval.imag)) {
-			if (Py_IS_NAN(v->cval.imag))
-				strncpy(buf, "nan*j", 6);
-			else if (copysign(1, v->cval.imag) == 1)
-				strncpy(buf, "inf*j", 6);
-			else
-				strncpy(buf, "-inf*j", 7);
-		}
-		else {
-			PyOS_snprintf(format, sizeof(format), "%%.%ig", precision);
-			PyOS_ascii_formatd(buf, bufsz - 1, format, v->cval.imag);
-			strncat(buf, "j", 1);
-		}
-	} else {
-		char re[64], im[64];
-		/* Format imaginary part with sign, real part without */
-		if (!Py_IS_FINITE(v->cval.real)) {
-			if (Py_IS_NAN(v->cval.real))
-				strncpy(re, "nan", 4);
-			/* else if (copysign(1, v->cval.real) == 1) */
-			else if (v->cval.real > 0)
-				strncpy(re, "inf", 4);
-			else
-				strncpy(re, "-inf", 5);
-		}
-		else {
-			PyOS_snprintf(format, sizeof(format), "%%.%ig", precision);
-			PyOS_ascii_formatd(re, sizeof(re), format, v->cval.real);
-		}
-		if (!Py_IS_FINITE(v->cval.imag)) {
-			if (Py_IS_NAN(v->cval.imag))
-				strncpy(im, "+nan*", 6);
-			/* else if (copysign(1, v->cval.imag) == 1) */
-			else if (v->cval.imag > 0)
-				strncpy(im, "+inf*", 6);
-			else
-				strncpy(im, "-inf*", 6);
-		}
-		else {
-			PyOS_snprintf(format, sizeof(format), "%%+.%ig", precision);
-			PyOS_ascii_formatd(im, sizeof(im), format, v->cval.imag);
-		}
-		PyOS_snprintf(buf, bufsz, "(%s%sj)", re, im);
-	}
+    PyObject *result = NULL;
+    Py_ssize_t len;
+
+    /* If these are non-NULL, they'll need to be freed. */
+    char *pre = NULL;
+    char *pim = NULL;
+    char *buf = NULL;
+
+    /* These do not need to be freed. They're either aliases for pim
+       and pre, or pointers to constants. */
+    char *re = NULL;
+    char *im = NULL;
+    char *lead = "";
+    char *tail = "";
+
+
+    if (v->cval.real == 0.) {
+        re = "";
+        if (!Py_IS_FINITE(v->cval.imag)) {
+            if (Py_IS_NAN(v->cval.imag))
+                im = "nan*";
+            else if (copysign(1, v->cval.imag) == 1)
+                im = "inf*";
+            else
+                im = "-inf*";
+        }
+        else {
+            pim = PyOS_double_to_string(v->cval.imag, format_code,
+                                        0, 0, NULL);
+            if (!pim) {
+                PyErr_NoMemory();
+                goto done;
+            }
+            im = pim;
+        }
+    } else {
+        /* Format imaginary part with sign, real part without */
+        if (!Py_IS_FINITE(v->cval.real)) {
+            if (Py_IS_NAN(v->cval.real))
+                re = "nan";
+            /* else if (copysign(1, v->cval.real) == 1) */
+            else if (v->cval.real > 0)
+                re = "inf";
+            else
+                re = "-inf";
+        }
+        else {
+            pre = PyOS_double_to_string(v->cval.real, format_code,
+                                        0, 0, NULL);
+            if (!pre) {
+                PyErr_NoMemory();
+                goto done;
+            }
+            re = pre;
+        }
+
+        if (!Py_IS_FINITE(v->cval.imag)) {
+            if (Py_IS_NAN(v->cval.imag))
+                im = "+nan*";
+            /* else if (copysign(1, v->cval.imag) == 1) */
+            else if (v->cval.imag > 0)
+                im = "+inf*";
+            else
+                im = "-inf*";
+        }
+        else {
+            pim = PyOS_double_to_string(v->cval.imag, format_code,
+                                        0, Py_DTSF_SIGN, NULL);
+            if (!pim) {
+                PyErr_NoMemory();
+                goto done;
+            }
+            im = pim;
+        }
+        lead = "(";
+        tail = ")";
+    }
+    /* Alloc the final buffer. Add one for the "j" in the format string, and
+       one for the trailing zero. */
+    len = strlen(lead) + strlen(re) + strlen(im) + strlen(tail) + 2;
+    buf = PyMem_Malloc(len);
+    if (!buf) {
+        PyErr_NoMemory();
+        goto done;
+    }
+    PyOS_snprintf(buf, len, "%s%s%sj%s", lead, re, im, tail);
+    result = PyUnicode_FromString(buf);
+done:
+    PyMem_Free(pim);
+    PyMem_Free(pre);
+    PyMem_Free(buf);
+
+    return result;
 }
 
 static PyObject *
 complex_repr(PyComplexObject *v)
 {
-	char buf[100];
-	complex_to_buf(buf, sizeof(buf), v, PREC_REPR);
-	return PyUnicode_FromString(buf);
+    return complex_format(v, 'r');
 }
 
 static PyObject *
 complex_str(PyComplexObject *v)
 {
-	char buf[100];
-	complex_to_buf(buf, sizeof(buf), v, PREC_STR);
-	return PyUnicode_FromString(buf);
+    return complex_format(v, 's');
 }
 
 static long

Objects/floatobject.c

@@ -197,8 +197,7 @@ PyFloat_FromString(PyObject *v)
 	sp = s;
 	/* We don't care about overflow or underflow.  If the platform supports
 	 * them, infinities and signed zeroes (on underflow) are fine.
-	 * However, strtod can return 0 for denormalized numbers, where atof
-	 * does not.  So (alas!) we special-case a zero result.  Note that
+	 * However, strtod can return 0 for denormalized numbers.  Note that
 	 * whether strtod sets errno on underflow is not defined, so we can't
 	 * key off errno.
          */
@@ -259,14 +258,6 @@ PyFloat_FromString(PyObject *v)
 				"null byte in argument for float()");
 		goto error;
 	}
-	if (x == 0.0) {
-		/* See above -- may have been strtod being anal
-		   about denorms. */
-		PyFPE_START_PROTECT("atof", goto error)
-		x = PyOS_ascii_atof(s);
-		PyFPE_END_PROTECT(x)
-		errno = 0;    /* whether atof ever set errno is undefined */
-	}
 	result = PyFloat_FromDouble(x);
   error:
 	if (s_buffer)
@@ -320,72 +311,6 @@ PyFloat_AsDouble(PyObject *op)
 	return val;
 }
 
-/* Methods */
-
-static void
-format_double(char *buf, size_t buflen, double ob_fval, int precision)
-{
-	register char *cp;
-	char format[32];
-	int i;
-
-	/* Subroutine for float_repr, float_str and float_print.
-	   We want float numbers to be recognizable as such,
-	   i.e., they should contain a decimal point or an exponent.
-	   However, %g may print the number as an integer;
-	   in such cases, we append ".0" to the string. */
-
-	PyOS_snprintf(format, 32, "%%.%ig", precision);
-	PyOS_ascii_formatd(buf, buflen, format, ob_fval);
-	cp = buf;
-	if (*cp == '-')
-		cp++;
-	for (; *cp != '\0'; cp++) {
-		/* Any non-digit means it's not an integer;
-		   this takes care of NAN and INF as well. */
-		if (!isdigit(Py_CHARMASK(*cp)))
-			break;
-	}
-	if (*cp == '\0') {
-		*cp++ = '.';
-		*cp++ = '0';
-		*cp++ = '\0';
-		return;
-	}
-	/* Checking the next three chars should be more than enough to
-	 * detect inf or nan, even on Windows. We check for inf or nan
-	 * at last because they are rare cases.
-	 */
-	for (i=0; *cp != '\0' && i<3; cp++, i++) {
-		if (isdigit(Py_CHARMASK(*cp)) || *cp == '.')
-			continue;
-		/* found something that is neither a digit nor point
-		 * it might be a NaN or INF
-		 */
-#ifdef Py_NAN
-		if (Py_IS_NAN(ob_fval)) {
-			strcpy(buf, "nan");
-		}
-                else
-#endif
-		if (Py_IS_INFINITY(ob_fval)) {
-			cp = buf;
-			if (*cp == '-')
-				cp++;
-			strcpy(cp, "inf");
-		}
-		break;
-	}
-
-}
-
-static void
-format_float(char *buf, size_t buflen, PyFloatObject *v, int precision)
-{
-	assert(PyFloat_Check(v));
-	format_double(buf, buflen, PyFloat_AS_DOUBLE(v), precision);
-}
-
 /* Macro and helper that convert PyObject obj to a C double and store
    the value in dbl.  If conversion to double raises an exception, obj is
    set to NULL, and the function invoking this macro returns NULL.  If
@@ -398,6 +323,8 @@ format_float(char *buf, size_t buflen, PyFloatObject *v, int precision)
 	else if (convert_to_double(&(obj), &(dbl)) < 0)	\
 		return obj;
 
+/* Methods */
+
 static int
 convert_to_double(PyObject **v, double *dbl)
 {
@@ -418,38 +345,30 @@ convert_to_double(PyObject **v, double *dbl)
 	return 0;
 }
 
-/* Precisions used by repr() and str(), respectively.
-
-   The repr() precision (17 significant decimal digits) is the minimal number
-   that is guaranteed to have enough precision so that if the number is read
-   back in the exact same binary value is recreated.  This is true for IEEE
-   floating point by design, and also happens to work for all other modern
-   hardware.
-
-   The str() precision is chosen so that in most cases, the rounding noise
-   created by various operations is suppressed, while giving plenty of
-   precision for practical use.
-
-*/
-
-#define PREC_REPR	17
-#define PREC_STR	12
+static PyObject *
+float_str_or_repr(PyFloatObject *v, char format_code)
+{
+    PyObject *result;
+    char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
+                                      format_code, 0, Py_DTSF_ADD_DOT_0,
+                                      NULL);
+    if (!buf)
+        return PyErr_NoMemory();
+    result = PyUnicode_FromString(buf);
+    PyMem_Free(buf);
+    return result;
+}
 
 static PyObject *
 float_repr(PyFloatObject *v)
 {
-	char buf[100];
-	format_float(buf, sizeof(buf), v, PREC_REPR);
-
-	return PyUnicode_FromString(buf);
+    return float_str_or_repr(v, 'r');
 }
 
 static PyObject *
 float_str(PyFloatObject *v)
 {
-	char buf[100];
-	format_float(buf, sizeof(buf), v, PREC_STR);
-	return PyUnicode_FromString(buf);
+    return float_str_or_repr(v, 's');
 }
 
 /* Comparison is pretty much a nightmare.  When comparing float to float,
@@ -1980,15 +1899,21 @@ PyFloat_Fini(void)
 			     i++, p++) {
 				if (PyFloat_CheckExact(p) &&
 				    Py_REFCNT(p) != 0) {
-					char buf[100];
-					format_float(buf, sizeof(buf), p, PREC_STR);
-					/* XXX(twouters) cast refcount to
-					   long until %zd is universally
-					   available
-					 */
-					fprintf(stderr,
+					char *buf = PyOS_double_to_string(
+						PyFloat_AS_DOUBLE(p), 'r',
+						0, 0, NULL);
+					if (buf) {
+						/* XXX(twouters) cast
+						   refcount to long
+						   until %zd is
+						   universally
+						   available
+						*/
+						fprintf(stderr,
 			     "#   <float at %p, refcnt=%ld, val=%s>\n",
 						p, (long)Py_REFCNT(p), buf);
+						PyMem_Free(buf);
+					}
 				}
 			}
 			list = list->next;
@@ -2233,14 +2158,6 @@ _PyFloat_Pack8(double x, unsigned char *p, int le)
 	}
 }
 
-/* Should only be used by marshal. */
-int
-_PyFloat_Repr(double x, char *p, size_t len)
-{
-	format_double(p, len, x, PREC_REPR);
-	return (int)strlen(p);
-}
-
 double
 _PyFloat_Unpack4(const unsigned char *p, int le)
 {

Objects/unicodeobject.c

@@ -8792,42 +8792,13 @@ getnextarg(PyObject *args, Py_ssize_t arglen, Py_ssize_t *p_argidx)
     return NULL;
 }
 
-static Py_ssize_t
-strtounicode(Py_UNICODE *buffer, const char *charbuffer)
+static void
+strtounicode(Py_UNICODE *buffer, const char *charbuffer, Py_ssize_t len)
 {
     register Py_ssize_t i;
-    Py_ssize_t len = strlen(charbuffer);
     for (i = len - 1; i >= 0; i--)
         buffer[i] = (Py_UNICODE) charbuffer[i];
-
-    return len;
-}
-
-static int
-doubletounicode(Py_UNICODE *buffer, size_t len, const char *format, double x)
-{
-    Py_ssize_t result;
-
-    PyOS_ascii_formatd((char *)buffer, len, format, x);
-    result = strtounicode(buffer, (char *)buffer);
-    return Py_SAFE_DOWNCAST(result, Py_ssize_t, int);
-}
-
-#if 0
-static int
-longtounicode(Py_UNICODE *buffer, size_t len, const char *format, long x)
-{
-    Py_ssize_t result;
-
-    PyOS_snprintf((char *)buffer, len, format, x);
-    result = strtounicode(buffer, (char *)buffer);
-    return Py_SAFE_DOWNCAST(result, Py_ssize_t, int);
 }
-#endif
-
-/* XXX To save some code duplication, formatfloat/long/int could have been
-   shared with stringobject.c, converting from 8-bit to Unicode after the
-   formatting is done. */
 
 static int
 formatfloat(Py_UNICODE *buf,
@@ -8837,54 +8808,59 @@ formatfloat(Py_UNICODE *buf,
             int type,
             PyObject *v)
 {
-    /* fmt = '%#.' + `prec` + `type`
-       worst case length = 3 + 10 (len of INT_MAX) + 1 = 14 (use 20)*/
-    char fmt[20];
+    /* eric.smith: To minimize disturbances in PyUnicode_Format (the
+       only caller of this routine), I'm going to keep the existing
+       API to this function. That means that we'll allocate memory and
+       then copy back into the supplied buffer. But that's better than
+       all of the changes that would be required in PyUnicode_Format
+       because it does lots of memory management tricks. */
+
+    char* p = NULL;
+    int result = -1;
     double x;
+    Py_ssize_t len;
 
     x = PyFloat_AsDouble(v);
     if (x == -1.0 && PyErr_Occurred())
-        return -1;
+        goto done;
     if (prec < 0)
         prec = 6;
+
     /* make sure that the decimal representation of precision really does
        need at most 10 digits: platforms with sizeof(int) == 8 exist! */
     if (prec > 0x7fffffffL) {
         PyErr_SetString(PyExc_OverflowError,
                         "outrageously large precision "
                         "for formatted float");
-        return -1;
+        goto done;
     }
 
     if (type == 'f' && fabs(x) >= 1e50)
         type = 'g';
-    /* Worst case length calc to ensure no buffer overrun:
-
-       'g' formats:
-       fmt = %#.<prec>g
-       buf = '-' + [0-9]*prec + '.' + 'e+' + (longest exp
-       for any double rep.)
-       len = 1 + prec + 1 + 2 + 5 = 9 + prec
-
-       'f' formats:
-       buf = '-' + [0-9]*x + '.' + [0-9]*prec (with x < 50)
-       len = 1 + 50 + 1 + prec = 52 + prec
 
-       If prec=0 the effective precision is 1 (the leading digit is
-       always given), therefore increase the length by one.
-
-    */
     if (((type == 'g' || type == 'G') &&
          buflen <= (size_t)10 + (size_t)prec) ||
-        (type == 'f' && buflen <= (size_t)53 + (size_t)prec)) {
+        ((type == 'f' || type == 'F') &&
+         buflen <= (size_t)53 + (size_t)prec)) {
         PyErr_SetString(PyExc_OverflowError,
                         "formatted float is too long (precision too large?)");
-        return -1;
+        goto done;
+    }
+
+    p = PyOS_double_to_string(x, type, prec,
+                              (flags & F_ALT) ? Py_DTSF_ALT : 0, NULL);
+    len = strlen(p);
+    if (len+1 >= buflen) {
+        /* Caller supplied buffer is not large enough. */
+        PyErr_NoMemory();
+        goto done;
     }
-    PyOS_snprintf(fmt, sizeof(fmt), "%%%s.%d%c",
-                  (flags&F_ALT) ? "#" : "",
-                  prec, type);
-    return doubletounicode(buf, buflen, fmt, x);
+    strtounicode(buf, p, len);
+    result = Py_SAFE_DOWNCAST(len, Py_ssize_t, int);
+
+done:
+    PyMem_Free(p);
+    return result;
 }
 
 static PyObject*
@@ -8903,84 +8879,6 @@ formatlong(PyObject *val, int flags, int prec, int type)
     return result;
 }
 
-#if 0
-static int
-formatint(Py_UNICODE *buf,
-          size_t buflen,
-          int flags,
-          int prec,
-          int type,
-          PyObject *v)
-{
-    /* fmt = '%#.' + `prec` + 'l' + `type`
-     * worst case length = 3 + 19 (worst len of INT_MAX on 64-bit machine)
-     *                     + 1 + 1
-     *                   = 24
-     */
-    char fmt[64]; /* plenty big enough! */
-    char *sign;
-    long x;
-
-    x = PyLong_AsLong(v);
-    if (x == -1 && PyErr_Occurred())
-        return -1;
-    if (x < 0 && type == 'u') {
-        type = 'd';
-    }
-    if (x < 0 && (type == 'x' || type == 'X' || type == 'o'))
-        sign = "-";
-    else
-        sign = "";
-    if (prec < 0)
-        prec = 1;
-
-    /* buf = '+'/'-'/'' + '0'/'0x'/'' + '[0-9]'*max(prec, len(x in octal))
-     * worst case buf = '-0x' + [0-9]*prec, where prec >= 11
-     */
-    if (buflen <= 14 || buflen <= (size_t)3 + (size_t)prec) {
-        PyErr_SetString(PyExc_OverflowError,
-                        "formatted integer is too long (precision too large?)");
-        return -1;
-    }
-
-    if ((flags & F_ALT) &&
-        (type == 'x' || type == 'X' || type == 'o')) {
-        /* When converting under %#o, %#x or %#X, there are a number
-         * of issues that cause pain:
-         * - for %#o, we want a different base marker than C
-         * - when 0 is being converted, the C standard leaves off
-         *   the '0x' or '0X', which is inconsistent with other
-         *   %#x/%#X conversions and inconsistent with Python's
-         *   hex() function
-         * - there are platforms that violate the standard and
-         *   convert 0 with the '0x' or '0X'
-         *   (Metrowerks, Compaq Tru64)
-         * - there are platforms that give '0x' when converting
-         *   under %#X, but convert 0 in accordance with the
-         *   standard (OS/2 EMX)
-         *
-         * We can achieve the desired consistency by inserting our
-         * own '0x' or '0X' prefix, and substituting %x/%X in place
-         * of %#x/%#X.
-         *
-         * Note that this is the same approach as used in
-         * formatint() in stringobject.c
-         */
-        PyOS_snprintf(fmt, sizeof(fmt), "%s0%c%%.%dl%c",
-                      sign, type, prec, type);
-    }
-    else {
-        PyOS_snprintf(fmt, sizeof(fmt), "%s%%%s.%dl%c",
-                      sign, (flags&F_ALT) ? "#" : "",
-                      prec, type);
-    }
-    if (sign[0])
-        return longtounicode(buf, buflen, fmt, -x);
-    else
-        return longtounicode(buf, buflen, fmt, x);
-}
-#endif
-
 static int
 formatchar(Py_UNICODE *buf,
            size_t buflen,
@@ -9359,8 +9257,6 @@ PyObject *PyUnicode_Format(PyObject *format,
             case 'F':
             case 'g':
             case 'G':
-                if (c == 'F')
-                    c = 'f';
                 pbuf = formatbuf;
                 len = formatfloat(pbuf, sizeof(formatbuf)/sizeof(Py_UNICODE),
                                   flags, prec, c, v);

Python/marshal.c

@@ -236,12 +236,15 @@ w_object(PyObject *v, WFILE *p)
 			w_string((char*)buf, 8, p);
 		}
 		else {
-			char buf[256]; /* Plenty to format any double */
-			n = _PyFloat_Repr(PyFloat_AS_DOUBLE(v),
-					  buf, sizeof(buf));
+			char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
+                                                          'r', 0, 0, NULL);
+			if (!buf)
+                            return;
+			n = strlen(buf);
 			w_byte(TYPE_FLOAT, p);
 			w_byte((int)n, p);
 			w_string(buf, (int)n, p);
+			PyMem_Free(buf);
 		}
 	}
 #ifndef WITHOUT_COMPLEX
@@ -263,17 +266,24 @@ w_object(PyObject *v, WFILE *p)
 			w_string((char*)buf, 8, p);
 		}
 		else {
-			char buf[256]; /* Plenty to format any double */
+			char *buf;
 			w_byte(TYPE_COMPLEX, p);
-			n = _PyFloat_Repr(PyComplex_RealAsDouble(v),
-					  buf, sizeof(buf));
+			buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
+                                                    'r', 0, 0, NULL);
+			if (!buf)
+                            return;
 			n = strlen(buf);
 			w_byte((int)n, p);
 			w_string(buf, (int)n, p);
-			n = _PyFloat_Repr(PyComplex_ImagAsDouble(v),
-					  buf, sizeof(buf));
+			PyMem_Free(buf);
+			buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
+                                                    'r', 0, 0, NULL);
+			if (!buf)
+                            return;
+			n = strlen(buf);
 			w_byte((int)n, p);
 			w_string(buf, (int)n, p);
+			PyMem_Free(buf);
 		}
 	}
 #endif