Concatenation on a long string breaks (SF #1526585).

Lib/test/regrtest.py

@@ -66,7 +66,9 @@ reports are written to.  These parameters all have defaults (5, 4 and
 
 -M runs tests that require an exorbitant amount of memory. These tests
 typically try to ascertain containers keep working when containing more than
-2 bilion objects, and only work on 64-bit systems. The passed-in memlimit,
+2 billion objects, which only works on 64-bit systems. There are also some
+tests that try to exhaust the address space of the process, which only makes
+sense on 32-bit systems with at least 2Gb of memory. The passed-in memlimit,
 which is a string in the form of '2.5Gb', determines howmuch memory the
 tests will limit themselves to (but they may go slightly over.) The number
 shouldn't be more memory than the machine has (including swap memory). You

Lib/test/test_bigaddrspace.py

+from test import test_support
+from test.test_support import bigaddrspacetest, MAX_Py_ssize_t
+
+import unittest
+import operator
+import sys
+
+
+class StrTest(unittest.TestCase):
+
+    @bigaddrspacetest
+    def test_concat(self):
+        s1 = 'x' * MAX_Py_ssize_t
+        self.assertRaises(OverflowError, operator.add, s1, '?')
+
+    @bigaddrspacetest
+    def test_optimized_concat(self):
+        x = 'x' * MAX_Py_ssize_t
+        try:
+            x = x + '?'     # this statement uses a fast path in ceval.c
+        except OverflowError:
+            pass
+        else:
+            self.fail("should have raised OverflowError")
+        try:
+            x += '?'        # this statement uses a fast path in ceval.c
+        except OverflowError:
+            pass
+        else:
+            self.fail("should have raised OverflowError")
+        self.assertEquals(len(x), MAX_Py_ssize_t)
+
+    ### the following test is pending a patch
+    #   (http://mail.python.org/pipermail/python-dev/2006-July/067774.html)
+    #@bigaddrspacetest
+    #def test_repeat(self):
+    #    self.assertRaises(OverflowError, operator.mul, 'x', MAX_Py_ssize_t + 1)
+
+
+def test_main():
+    test_support.run_unittest(StrTest)
+
+if __name__ == '__main__':
+    if len(sys.argv) > 1:
+        test_support.set_memlimit(sys.argv[1])
+    test_main()

Lib/test/test_support.py

@@ -314,6 +314,12 @@ _1M = 1024*1024
 _1G = 1024 * _1M
 _2G = 2 * _1G
 
+# Hack to get at the maximum value an internal index can take.
+class _Dummy:
+    def __getslice__(self, i, j):
+        return j
+MAX_Py_ssize_t = _Dummy()[:]
+
 def set_memlimit(limit):
     import re
     global max_memuse
@@ -328,7 +334,9 @@ def set_memlimit(limit):
     if m is None:
         raise ValueError('Invalid memory limit %r' % (limit,))
     memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()])
-    if memlimit < 2.5*_1G:
+    if memlimit > MAX_Py_ssize_t:
+        memlimit = MAX_Py_ssize_t
+    if memlimit < _2G - 1:
         raise ValueError('Memory limit %r too low to be useful' % (limit,))
     max_memuse = memlimit
 
@@ -371,6 +379,17 @@ def bigmemtest(minsize, memuse, overhead=5*_1M):
         return wrapper
     return decorator
 
+def bigaddrspacetest(f):
+    """Decorator for tests that fill the address space."""
+    def wrapper(self):
+        if max_memuse < MAX_Py_ssize_t:
+            if verbose:
+                sys.stderr.write("Skipping %s because of memory "
+                                 "constraint\n" % (f.__name__,))
+        else:
+            return f(self)
+    return wrapper
+
 #=======================================================================
 # Preliminary PyUNIT integration.
 

Python/ceval.c

@@ -4225,6 +4225,14 @@ string_concatenate(PyObject *v, PyObject *w,
 {
 	/* This function implements 'variable += expr' when both arguments
 	   are strings. */
+	Py_ssize_t v_len = PyString_GET_SIZE(v);
+	Py_ssize_t w_len = PyString_GET_SIZE(w);
+	Py_ssize_t new_len = v_len + w_len;
+	if (new_len < 0) {
+		PyErr_SetString(PyExc_OverflowError,
+				"strings are too large to concat");
+		return NULL;
+	}
 
 	if (v->ob_refcnt == 2) {
 		/* In the common case, there are 2 references to the value
@@ -4269,9 +4277,7 @@ string_concatenate(PyObject *v, PyObject *w,
 		/* Now we own the last reference to 'v', so we can resize it
 		 * in-place.
 		 */
-		Py_ssize_t v_len = PyString_GET_SIZE(v);
-		Py_ssize_t w_len = PyString_GET_SIZE(w);
-		if (_PyString_Resize(&v, v_len + w_len) != 0) {
+		if (_PyString_Resize(&v, new_len) != 0) {
 			/* XXX if _PyString_Resize() fails, 'v' has been
 			 * deallocated so it cannot be put back into 'variable'.
 			 * The MemoryError is raised when there is no value in