Merged revisions 87968,87971-87975 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/branches/py3k

........
  r87968 | antoine.pitrou | 2011-01-12 21:46:37 +0100 (mer., 12 janv. 2011) | 4 lines

  Fix the expected memory use of utf-8 encoding.  Also, release the
  one reference to a huge object even when an exception is raised.
........
  r87971 | antoine.pitrou | 2011-01-12 22:19:59 +0100 (mer., 12 janv. 2011) | 3 lines

  Make test skipping message nicer, and remove the rather useless "overhead" parameter.
........
  r87972 | antoine.pitrou | 2011-01-12 22:40:20 +0100 (mer., 12 janv. 2011) | 3 lines

  Fix @bigmemtest when no limit is given by the user (oops)
........
  r87973 | antoine.pitrou | 2011-01-12 22:50:44 +0100 (mer., 12 janv. 2011) | 3 lines

  More informative skip message in @bigaddrspace
........
  r87974 | antoine.pitrou | 2011-01-12 22:58:39 +0100 (mer., 12 janv. 2011) | 3 lines

  A better message again
........
  r87975 | antoine.pitrou | 2011-01-12 23:02:45 +0100 (mer., 12 janv. 2011) | 3 lines

  Fix test_bigaddrspace (some tests didn't trigger the expected MemoryError)
........

Lib/test/support.py

@@ -784,13 +784,12 @@ def set_memlimit(limit):
         raise ValueError('Memory limit %r too low to be useful' % (limit,))
     max_memuse = memlimit
 
-def bigmemtest(minsize, memuse, overhead=5*_1M):
+def bigmemtest(minsize, memuse):
     """Decorator for bigmem tests.
 
     'minsize' is the minimum useful size for the test (in arbitrary,
     test-interpreted units.) 'memuse' is the number of 'bytes per size' for
-    the test, or a good estimate of it. 'overhead' specifies fixed overhead,
-    independent of the testsize, and defaults to 5Mb.
+    the test, or a good estimate of it.
 
     The decorator tries to guess a good value for 'size' and passes it to
     the decorated test function. If minsize * memuse is more than the
@@ -802,52 +801,43 @@ def bigmemtest(minsize, memuse, overhead=5*_1M):
             # Retrieve values in case someone decided to adjust them
             minsize = wrapper.minsize
             memuse = wrapper.memuse
-            overhead = wrapper.overhead
             if not max_memuse:
                 # If max_memuse is 0 (the default),
                 # we still want to run the tests with size set to a few kb,
                 # to make sure they work. We still want to avoid using
                 # too much memory, though, but we do that noisily.
                 maxsize = 5147
-                self.assertFalse(maxsize * memuse + overhead > 20 * _1M)
+                self.assertFalse(maxsize * memuse > 20 * _1M)
             else:
-                maxsize = int((max_memuse - overhead) / memuse)
+                maxsize = int(max_memuse / memuse)
                 if maxsize < minsize:
-                    # Really ought to print 'test skipped' or something
-                    if verbose:
-                        sys.stderr.write("Skipping %s because of memory "
-                                         "constraint\n" % (f.__name__,))
-                    return
-                # Try to keep some breathing room in memory use
-                maxsize = max(maxsize - 50 * _1M, minsize)
+                    raise unittest.SkipTest(
+                        "not enough memory: %.1fG minimum needed"
+                        % (minsize * memuse / (1024 ** 3)))
             return f(self, maxsize)
         wrapper.minsize = minsize
         wrapper.memuse = memuse
-        wrapper.overhead = overhead
         return wrapper
     return decorator
 
-def precisionbigmemtest(size, memuse, overhead=5*_1M):
+def precisionbigmemtest(size, memuse):
     def decorator(f):
         def wrapper(self):
             size = wrapper.size
             memuse = wrapper.memuse
-            overhead = wrapper.overhead
             if not real_max_memuse:
                 maxsize = 5147
             else:
                 maxsize = size
 
                 if real_max_memuse and real_max_memuse < maxsize * memuse:
-                    if verbose:
-                        sys.stderr.write("Skipping %s because of memory "
-                                         "constraint\n" % (f.__name__,))
-                    return
+                    raise unittest.SkipTest(
+                        "not enough memory: %.1fG minimum needed"
+                        % (size * memuse / (1024 ** 3)))
 
             return f(self, maxsize)
         wrapper.size = size
         wrapper.memuse = memuse
-        wrapper.overhead = overhead
         return wrapper
     return decorator
 
@@ -855,9 +845,13 @@ def bigaddrspacetest(f):
     """Decorator for tests that fill the address space."""
     def wrapper(self):
         if max_memuse < MAX_Py_ssize_t:
-            if verbose:
-                sys.stderr.write("Skipping %s because of memory "
-                                 "constraint\n" % (f.__name__,))
+            if MAX_Py_ssize_t >= 2**63 - 1 and max_memuse >= 2**31:
+                raise unittest.SkipTest(
+                    "not enough memory: try a 32-bit build instead")
+            else:
+                raise unittest.SkipTest(
+                    "not enough memory: %.1fG minimum needed"
+                    % (MAX_Py_ssize_t / (1024 ** 3)))
         else:
             return f(self)
     return wrapper

Lib/test/test_bigaddrspace.py

@@ -23,25 +23,34 @@ class BytesTest(unittest.TestCase):
         # Allocate a bytestring that's near the maximum size allowed by
         # the address space, and then try to build a new, larger one through
         # concatenation.
-        x = b"x" * (MAX_Py_ssize_t - 128)
-        self.assertRaises(OverflowError, operator.add, x, b"x" * 128)
+        try:
+            x = b"x" * (MAX_Py_ssize_t - 128)
+            self.assertRaises(OverflowError, operator.add, x, b"x" * 128)
+        finally:
+            x = None
 
     @bigaddrspacetest
     def test_optimized_concat(self):
-        x = b"x" * (MAX_Py_ssize_t - 128)
+        try:
+            x = b"x" * (MAX_Py_ssize_t - 128)
 
-        with self.assertRaises(OverflowError) as cm:
-            # this statement uses a fast path in ceval.c
-            x = x + b"x" * 128
+            with self.assertRaises(OverflowError) as cm:
+                # this statement used a fast path in ceval.c
+                x = x + b"x" * 128
 
-        with self.assertRaises(OverflowError) as cm:
-            # this statement uses a fast path in ceval.c
-            x +=  b"x" * 128
+            with self.assertRaises(OverflowError) as cm:
+                # this statement used a fast path in ceval.c
+                x +=  b"x" * 128
+        finally:
+            x = None
 
     @bigaddrspacetest
     def test_repeat(self):
-        x = b"x" * (MAX_Py_ssize_t - 128)
-        self.assertRaises(OverflowError, operator.mul, x, 128)
+        try:
+            x = b"x" * (MAX_Py_ssize_t - 128)
+            self.assertRaises(OverflowError, operator.mul, x, 128)
+        finally:
+            x = None
 
 
 class StrTest(unittest.TestCase):
@@ -50,28 +59,37 @@ class StrTest(unittest.TestCase):
 
     @bigaddrspacetest
     def test_concat(self):
-        # Create a string half the size that would fill the address space
-        x = "x" * (MAX_Py_ssize_t // (2 * self.unicodesize))
-        # Unicode objects trigger MemoryError in case an operation that's
-        # going to cause a size overflow is executed
-        self.assertRaises(MemoryError, operator.add, x, x)
+        try:
+            # Create a string that would fill almost the address space
+            x = "x" * int(MAX_Py_ssize_t // (1.1 * self.unicodesize))
+            # Unicode objects trigger MemoryError in case an operation that's
+            # going to cause a size overflow is executed
+            self.assertRaises(MemoryError, operator.add, x, x)
+        finally:
+            x = None
 
     @bigaddrspacetest
     def test_optimized_concat(self):
-        x = "x" * (MAX_Py_ssize_t // (2 * self.unicodesize))
+        try:
+            x = "x" * int(MAX_Py_ssize_t // (1.1 * self.unicodesize))
 
-        with self.assertRaises(MemoryError) as cm:
-            # this statement uses a fast path in ceval.c
-            x = x + x
+            with self.assertRaises(MemoryError) as cm:
+                # this statement uses a fast path in ceval.c
+                x = x + x
 
-        with self.assertRaises(MemoryError) as cm:
-            # this statement uses a fast path in ceval.c
-            x +=  x
+            with self.assertRaises(MemoryError) as cm:
+                # this statement uses a fast path in ceval.c
+                x +=  x
+        finally:
+            x = None
 
     @bigaddrspacetest
     def test_repeat(self):
-        x = "x" * (MAX_Py_ssize_t // (2 * self.unicodesize))
-        self.assertRaises(MemoryError, operator.mul, x, 2)
+        try:
+            x = "x" * int(MAX_Py_ssize_t // (1.1 * self.unicodesize))
+            self.assertRaises(MemoryError, operator.mul, x, 2)
+        finally:
+            x = None
 
 
 def test_main():

Lib/test/test_bigmem.py

@@ -564,8 +564,11 @@ class StrTest(unittest.TestCase, BaseStrTest):
         if expectedsize is None:
             expectedsize = size
 
-        s = c * size
-        self.assertEqual(len(s.encode(enc)), expectedsize)
+        try:
+            s = c * size
+            self.assertEqual(len(s.encode(enc)), expectedsize)
+        finally:
+            s = None
 
     def setUp(self):
         # HACK: adjust memory use of tests inherited from BaseStrTest
@@ -586,7 +589,8 @@ class StrTest(unittest.TestCase, BaseStrTest):
         for name, memuse in self._adjusted.items():
             getattr(type(self), name).memuse = memuse
 
-    @bigmemtest(minsize=_2G + 2, memuse=character_size + 1)
+    # the utf8 encoder preallocates big time (4x the number of characters)
+    @bigmemtest(minsize=_2G + 2, memuse=character_size + 4)
     def test_encode(self, size):
         return self.basic_encode_test(size, 'utf-8')