Merged revisions 77798 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r77798 | antoine.pitrou | 2010-01-27 21:59:50 +0100 (mer., 27 janv. 2010) | 8 lines

  Issue #7610: Reworked implementation of the internal
  :class:`zipfile.ZipExtFile` class used to represent files stored inside
  an archive.  The new implementation is significantly faster and can
  be wrapped in a :class:`io.BufferedReader` object for more speedups.
  It also solves an issue where interleaved calls to `read()` and
  `readline()` give wrong results.  Patch by Nir Aides.
........

Lib/test/test_zipfile.py

@@ -168,6 +168,45 @@ class TestsWithSourceFile(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_random_open_test(f, zipfile.ZIP_STORED)
 
+    def test_univeral_readaheads(self):
+        f = io.BytesIO()
+
+        data = b'a\r\n' * 16 * 1024
+        zipfp = zipfile.ZipFile(f, 'w', zipfile.ZIP_STORED)
+        zipfp.writestr(TESTFN, data)
+        zipfp.close()
+
+        data2 = b''
+        zipfp = zipfile.ZipFile(f, 'r')
+        zipopen = zipfp.open(TESTFN, 'rU')
+        for line in zipopen:
+            data2 += line
+        zipfp.close()
+
+        self.assertEqual(data, data2.replace(b'\n', b'\r\n'))
+
+    def zip_readline_read_test(self, f, compression):
+        self.make_test_archive(f, compression)
+
+        # Read the ZIP archive
+        zipfp = zipfile.ZipFile(f, "r")
+        zipopen = zipfp.open(TESTFN)
+
+        data = b''
+        while True:
+            read = zipopen.readline()
+            if not read:
+                break
+            data += read
+
+            read = zipopen.read(100)
+            if not read:
+                break
+            data += read
+
+        self.assertEqual(data, self.data)
+        zipfp.close()
+
     def zip_readline_test(self, f, compression):
         self.make_test_archive(f, compression)
 
@@ -195,6 +234,11 @@ class TestsWithSourceFile(unittest.TestCase):
             for line, zipline in zip(self.line_gen, zipfp.open(TESTFN)):
                 self.assertEqual(zipline, line + '\n')
 
+    def test_readline_read_stored(self):
+        # Issue #7610: calls to readline() interleaved with calls to read().
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_readline_read_test(f, zipfile.ZIP_STORED)
+
     def test_readline_stored(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_readline_test(f, zipfile.ZIP_STORED)
@@ -223,6 +267,12 @@ class TestsWithSourceFile(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.zip_random_open_test(f, zipfile.ZIP_DEFLATED)
 
+    @skipUnless(zlib, "requires zlib")
+    def test_readline_read_deflated(self):
+        # Issue #7610: calls to readline() interleaved with calls to read().
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.zip_readline_read_test(f, zipfile.ZIP_DEFLATED)
+
     @skipUnless(zlib, "requires zlib")
     def test_readline_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
@@ -1067,6 +1117,29 @@ class UniversalNewlineTests(unittest.TestCase):
                 zipdata = zipfp.open(fn, "rU").read()
                 self.assertEqual(self.arcdata[sep], zipdata)
 
+    def readline_read_test(self, f, compression):
+        self.make_test_archive(f, compression)
+
+        # Read the ZIP archive
+        zipfp = zipfile.ZipFile(f, "r")
+        for sep, fn in self.arcfiles.items():
+            zipopen = zipfp.open(fn, "rU")
+            data = b''
+            while True:
+                read = zipopen.readline()
+                if not read:
+                    break
+                data += read
+
+                read = zipopen.read(5)
+                if not read:
+                    break
+                data += read
+
+            self.assertEqual(data, self.arcdata['\n'])
+
+        zipfp.close()
+
     def readline_test(self, f, compression):
         self.make_test_archive(f, compression)
 
@@ -1101,6 +1174,11 @@ class UniversalNewlineTests(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.read_test(f, zipfile.ZIP_STORED)
 
+    def test_readline_read_stored(self):
+        # Issue #7610: calls to readline() interleaved with calls to read().
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.readline_read_test(f, zipfile.ZIP_STORED)
+
     def test_readline_stored(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.readline_test(f, zipfile.ZIP_STORED)
@@ -1118,6 +1196,12 @@ class UniversalNewlineTests(unittest.TestCase):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
             self.read_test(f, zipfile.ZIP_DEFLATED)
 
+    @skipUnless(zlib, "requires zlib")
+    def test_readline_read_deflated(self):
+        # Issue #7610: calls to readline() interleaved with calls to read().
+        for f in (TESTFN2, TemporaryFile(), io.BytesIO()):
+            self.readline_read_test(f, zipfile.ZIP_DEFLATED)
+
     @skipUnless(zlib, "requires zlib")
     def test_readline_deflated(self):
         for f in (TESTFN2, TemporaryFile(), io.BytesIO()):

Misc/NEWS

@@ -225,7 +225,6 @@ C-API
 - Issue #1419652: Change the first argument to PyImport_AppendInittab() to
   ``const char *`` as the string is stored beyond the call.
 
-
 - Issue #2422: When compiled with the ``--with-valgrind`` option, the
   pymalloc allocator will be automatically disabled when running under
   Valgrind.  This gives improved memory leak detection when running
@@ -234,6 +233,13 @@ C-API
 Library
 -------
 
+- Issue #7610: Reworked implementation of the internal
+  :class:`zipfile.ZipExtFile` class used to represent files stored inside
+  an archive.  The new implementation is significantly faster and can
+  be wrapped in a :class:`io.BufferedReader` object for more speedups.
+  It also solves an issue where interleaved calls to `read()` and
+  `readline()` give wrong results.  Patch by Nir Aides.
+
 - Issue #6963: Added "maxtasksperchild" argument to multiprocessing.Pool,
   allowing for a maximum number of tasks within the pool to be completed by
   the worker before that worker is terminated, and a new one created to