Skip to content

C
cpython
Commit b13a20f5 authored by Serhiy Storchaka's avatar Serhiy Storchaka Committed by GitHub
Browse files
Options

bpo-5438: Update memory requirements and optimize test_bigmem.py. (GH-11123)

parent 83dd4e87
Show whitespace changes
Inline Side-by-side
Showing with 69 additions and 68 deletions
Lib/test/test_bigmem.py
View file @ b13a20f5
......@@ -64,6 +64,7 @@ import sys
ascii_char_size = 1
ucs2_char_size = 2
ucs4_char_size = 4
pointer_size = 4 if sys.maxsize < 2**32 else 8
class BaseStrTest:
......@@ -372,7 +373,7 @@ class BaseStrTest:
# suffer for the list size. (Otherwise, it'd cost another 48 times
# size in bytes!) Nevertheless, a list of size takes
# 8*size bytes.
@bigmemtest(size=_2G + 5, memuse=2 * ascii_char_size + 8)
@bigmemtest(size=_2G + 5, memuse=ascii_char_size * 2 + pointer_size)
def test_split_large(self, size):
_ = self.from_latin1
s = _(' a') * size + _(' ')
......@@ -604,15 +605,15 @@ class StrTest(unittest.TestCase, BaseStrTest):
for name, memuse in self._adjusted.items():
getattr(type(self), name).memuse = memuse
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
def test_capitalize(self, size):
self._test_capitalize(size)
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
def test_title(self, size):
self._test_title(size)
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3)
@bigmemtest(size=_2G, memuse=ucs4_char_size * 3 + ascii_char_size * 2)
def test_swapcase(self, size):
self._test_swapcase(size)
......@@ -630,7 +631,7 @@ class StrTest(unittest.TestCase, BaseStrTest):
except MemoryError:
pass # acceptable on 32-bit
@bigmemtest(size=_4G // 5 + 70, memuse=ascii_char_size + ucs4_char_size + 1)
@bigmemtest(size=_4G // 5 + 70, memuse=ascii_char_size + 8 + 1)
def test_encode_utf7(self, size):
try:
return self.basic_encode_test(size, 'utf7')
......@@ -820,7 +821,7 @@ class TupleTest(unittest.TestCase):
# having more than 2<<31 references to any given object. Hence the
# use of different types of objects as contents in different tests.
@bigmemtest(size=_2G + 2, memuse=16)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2)
def test_compare(self, size):
t1 = ('',) * size
t2 = ('',) * size
......@@ -843,15 +844,15 @@ class TupleTest(unittest.TestCase):
t = t + t
self.assertEqual(len(t), size * 2)
@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
def test_concat_small(self, size):
return self.basic_concat_test(size)
@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
def test_concat_large(self, size):
return self.basic_concat_test(size)
@bigmemtest(size=_2G // 5 + 10, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 10, memuse=pointer_size * 5)
def test_contains(self, size):
t = (1, 2, 3, 4, 5) * size
self.assertEqual(len(t), size * 5)
......@@ -859,7 +860,7 @@ class TupleTest(unittest.TestCase):
self.assertFalse((1, 2, 3, 4, 5) in t)
self.assertFalse(0 in t)
@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_hash(self, size):
t1 = (0,) * size
h1 = hash(t1)
......@@ -867,7 +868,7 @@ class TupleTest(unittest.TestCase):
t2 = (0,) * (size + 1)
self.assertFalse(h1 == hash(t2))
@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_index_and_slice(self, size):
t = (None,) * size
self.assertEqual(len(t), size)
......@@ -892,11 +893,11 @@ class TupleTest(unittest.TestCase):
t = t * 2
self.assertEqual(len(t), size * 2)
@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
def test_repeat_small(self, size):
return self.basic_test_repeat(size)
@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
def test_repeat_large(self, size):
return self.basic_test_repeat(size)
......@@ -904,48 +905,42 @@ class TupleTest(unittest.TestCase):
def test_repeat_large_2(self, size):
return self.basic_test_repeat(size)
@bigmemtest(size=_1G - 1, memuse=9)
@bigmemtest(size=_1G - 1, memuse=pointer_size * 2)
def test_from_2G_generator(self, size):
self.skipTest("test needs much more memory than advertised, see issue5438")
try:
t = tuple(range(size))
t = tuple(iter([42]*size))
except MemoryError:
pass # acceptable on 32-bit
else:
count = 0
for item in t:
self.assertEqual(item, count)
count += 1
self.assertEqual(count, size)
self.assertEqual(len(t), size)
self.assertEqual(t[:10], (42,) * 10)
self.assertEqual(t[-10:], (42,) * 10)
@bigmemtest(size=_1G - 25, memuse=9)
@bigmemtest(size=_1G - 25, memuse=pointer_size * 2)
def test_from_almost_2G_generator(self, size):
self.skipTest("test needs much more memory than advertised, see issue5438")
try:
t = tuple(range(size))
count = 0
for item in t:
self.assertEqual(item, count)
count += 1
self.assertEqual(count, size)
t = tuple(iter([42]*size))
except MemoryError:
pass # acceptable, expected on 32-bit
pass # acceptable on 32-bit
else:
self.assertEqual(len(t), size)
self.assertEqual(t[:10], (42,) * 10)
self.assertEqual(t[-10:], (42,) * 10)
# Like test_concat, split in two.
def basic_test_repr(self, size):
t = (0,) * size
t = (False,) * size
s = repr(t)
# The repr of a tuple of 0's is exactly three times the tuple length.
self.assertEqual(len(s), size * 3)
self.assertEqual(s[:5], '(0, 0')
self.assertEqual(s[-5:], '0, 0)')
self.assertEqual(s.count('0'), size)
# The repr of a tuple of Falses is exactly 7 times the tuple length.
self.assertEqual(len(s), size * 7)
self.assertEqual(s[:10], '(False, Fa')
self.assertEqual(s[-10:], 'se, False)')
@bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size)
@bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7)
def test_repr_small(self, size):
return self.basic_test_repr(size)
@bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size)
@bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7)
def test_repr_large(self, size):
return self.basic_test_repr(size)
......@@ -956,7 +951,7 @@ class ListTest(unittest.TestCase):
# lists hold references to various objects to test their refcount
# limits.
@bigmemtest(size=_2G + 2, memuse=16)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2)
def test_compare(self, size):
l1 = [''] * size
l2 = [''] * size
......@@ -979,14 +974,16 @@ class ListTest(unittest.TestCase):
l = l + l
self.assertEqual(len(l), size * 2)
@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
def test_concat_small(self, size):
return self.basic_test_concat(size)
@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
def test_concat_large(self, size):
return self.basic_test_concat(size)
# XXX This tests suffers from overallocation, just like test_append.
# This should be fixed in future.
def basic_test_inplace_concat(self, size):
l = [sys.stdout] * size
l += l
......@@ -994,15 +991,15 @@ class ListTest(unittest.TestCase):
self.assertTrue(l[0] is l[-1])
self.assertTrue(l[size - 1] is l[size + 1])
@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
def test_inplace_concat_small(self, size):
return self.basic_test_inplace_concat(size)
@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
def test_inplace_concat_large(self, size):
return self.basic_test_inplace_concat(size)
@bigmemtest(size=_2G // 5 + 10, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 10, memuse=pointer_size * 5)
def test_contains(self, size):
l = [1, 2, 3, 4, 5] * size
self.assertEqual(len(l), size * 5)
......@@ -1010,12 +1007,12 @@ class ListTest(unittest.TestCase):
self.assertFalse([1, 2, 3, 4, 5] in l)
self.assertFalse(0 in l)
@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_hash(self, size):
l = [0] * size
self.assertRaises(TypeError, hash, l)
@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_index_and_slice(self, size):
l = [None] * size
self.assertEqual(len(l), size)
......@@ -1079,14 +1076,16 @@ class ListTest(unittest.TestCase):
l = l * 2
self.assertEqual(len(l), size * 2)
@bigmemtest(size=_2G // 2 + 2, memuse=24)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 3)
def test_repeat_small(self, size):
return self.basic_test_repeat(size)
@bigmemtest(size=_2G + 2, memuse=24)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 3)
def test_repeat_large(self, size):
return self.basic_test_repeat(size)
# XXX This tests suffers from overallocation, just like test_append.
# This should be fixed in future.
def basic_test_inplace_repeat(self, size):
l = ['']
l *= size
......@@ -1099,34 +1098,34 @@ class ListTest(unittest.TestCase):
self.assertEqual(len(l), size * 2)
self.assertTrue(l[size - 1] is l[-1])
@bigmemtest(size=_2G // 2 + 2, memuse=16)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
def test_inplace_repeat_small(self, size):
return self.basic_test_inplace_repeat(size)
@bigmemtest(size=_2G + 2, memuse=16)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
def test_inplace_repeat_large(self, size):
return self.basic_test_inplace_repeat(size)
def basic_test_repr(self, size):
l = [0] * size
l = [False] * size
s = repr(l)
# The repr of a list of 0's is exactly three times the list length.
self.assertEqual(len(s), size * 3)
self.assertEqual(s[:5], '[0, 0')
self.assertEqual(s[-5:], '0, 0]')
self.assertEqual(s.count('0'), size)
# The repr of a list of Falses is exactly 7 times the list length.
self.assertEqual(len(s), size * 7)
self.assertEqual(s[:10], '[False, Fa')
self.assertEqual(s[-10:], 'se, False]')
self.assertEqual(s.count('F'), size)
@bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size)
@bigmemtest(size=_2G // 7 + 2, memuse=pointer_size + ascii_char_size * 7)
def test_repr_small(self, size):
return self.basic_test_repr(size)
@bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size)
@bigmemtest(size=_2G + 2, memuse=pointer_size + ascii_char_size * 7)
def test_repr_large(self, size):
return self.basic_test_repr(size)
# list overallocates ~1/8th of the total size (on first expansion) so
# the single list.append call puts memuse at 9 bytes per size.
@bigmemtest(size=_2G, memuse=9)
@bigmemtest(size=_2G, memuse=pointer_size * 9/8)
def test_append(self, size):
l = [object()] * size
l.append(object())
......@@ -1134,12 +1133,14 @@ class ListTest(unittest.TestCase):
self.assertTrue(l[-3] is l[-2])
self.assertFalse(l[-2] is l[-1])
@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
def test_count(self, size):
l = [1, 2, 3, 4, 5] * size
self.assertEqual(l.count(1), size)
self.assertEqual(l.count("1"), 0)
# XXX This tests suffers from overallocation, just like test_append.
# This should be fixed in future.
def basic_test_extend(self, size):
l = [object] * size
l.extend(l)
......@@ -1147,15 +1148,15 @@ class ListTest(unittest.TestCase):
self.assertTrue(l[0] is l[-1])
self.assertTrue(l[size - 1] is l[size + 1])
@bigmemtest(size=_2G // 2 + 2, memuse=16)
@bigmemtest(size=_2G // 2 + 2, memuse=pointer_size * 2 * 9/8)
def test_extend_small(self, size):
return self.basic_test_extend(size)
@bigmemtest(size=_2G + 2, memuse=16)
@bigmemtest(size=_2G + 2, memuse=pointer_size * 2 * 9/8)
def test_extend_large(self, size):
return self.basic_test_extend(size)
@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
def test_index(self, size):
l = [1, 2, 3, 4, 5] * size
size *= 5
......@@ -1166,7 +1167,7 @@ class ListTest(unittest.TestCase):
self.assertRaises(ValueError, l.index, 6)
# This tests suffers from overallocation, just like test_append.
@bigmemtest(size=_2G + 10, memuse=9)
@bigmemtest(size=_2G + 10, memuse=pointer_size * 9/8)
def test_insert(self, size):
l = [1.0] * size
l.insert(size - 1, "A")
......@@ -1185,7 +1186,7 @@ class ListTest(unittest.TestCase):
self.assertEqual(l[:3], [1.0, "C", 1.0])
self.assertEqual(l[size - 3:], ["A", 1.0, "B"])
@bigmemtest(size=_2G // 5 + 4, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 4, memuse=pointer_size * 5)
def test_pop(self, size):
l = ["a", "b", "c", "d", "e"] * size
size *= 5
......@@ -1209,7 +1210,7 @@ class ListTest(unittest.TestCase):
self.assertEqual(item, "c")
self.assertEqual(l[-2:], ["b", "d"])
@bigmemtest(size=_2G + 10, memuse=8)
@bigmemtest(size=_2G + 10, memuse=pointer_size)
def test_remove(self, size):
l = [10] * size
self.assertEqual(len(l), size)
......@@ -1229,7 +1230,7 @@ class ListTest(unittest.TestCase):
self.assertEqual(len(l), size)
self.assertEqual(l[-2:], [10, 10])
@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5)
def test_reverse(self, size):
l = [1, 2, 3, 4, 5] * size
l.reverse()
......@@ -1237,7 +1238,7 @@ class ListTest(unittest.TestCase):
self.assertEqual(l[-5:], [5, 4, 3, 2, 1])
self.assertEqual(l[:5], [5, 4, 3, 2, 1])
@bigmemtest(size=_2G // 5 + 2, memuse=8 * 5)
@bigmemtest(size=_2G // 5 + 2, memuse=pointer_size * 5 * 1.5)
def test_sort(self, size):
l = [1, 2, 3, 4, 5] * size
l.sort()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7