Commit 77250f4d authored by Alexandre Vassalotti's avatar Alexandre Vassalotti

Added fast alternate io.BytesIO implementation and its test suite.

Removed old test suite for StringIO.
Modified truncate() to imply a seek to given argument value.
parent 5d8da20d
...@@ -490,6 +490,7 @@ class IOBase(metaclass=abc.ABCMeta): ...@@ -490,6 +490,7 @@ class IOBase(metaclass=abc.ABCMeta):
terminator(s) recognized. terminator(s) recognized.
""" """
# For backwards compatibility, a (slowish) readline(). # For backwards compatibility, a (slowish) readline().
self._checkClosed()
if hasattr(self, "peek"): if hasattr(self, "peek"):
def nreadahead(): def nreadahead():
readahead = self.peek(1) readahead = self.peek(1)
...@@ -531,7 +532,7 @@ class IOBase(metaclass=abc.ABCMeta): ...@@ -531,7 +532,7 @@ class IOBase(metaclass=abc.ABCMeta):
lines will be read if the total size (in bytes/characters) of all lines will be read if the total size (in bytes/characters) of all
lines so far exceeds hint. lines so far exceeds hint.
""" """
if hint is None: if hint is None or hint <= 0:
return list(self) return list(self)
n = 0 n = 0
lines = [] lines = []
...@@ -726,6 +727,8 @@ class _BufferedIOMixin(BufferedIOBase): ...@@ -726,6 +727,8 @@ class _BufferedIOMixin(BufferedIOBase):
if pos is None: if pos is None:
pos = self.tell() pos = self.tell()
# XXX: Should seek() be used, instead of passing the position
# XXX directly to truncate?
return self.raw.truncate(pos) return self.raw.truncate(pos)
### Flush and close ### ### Flush and close ###
...@@ -765,7 +768,7 @@ class _BufferedIOMixin(BufferedIOBase): ...@@ -765,7 +768,7 @@ class _BufferedIOMixin(BufferedIOBase):
return self.raw.isatty() return self.raw.isatty()
class BytesIO(BufferedIOBase): class _BytesIO(BufferedIOBase):
"""Buffered I/O implementation using an in-memory bytes buffer.""" """Buffered I/O implementation using an in-memory bytes buffer."""
...@@ -779,13 +782,19 @@ class BytesIO(BufferedIOBase): ...@@ -779,13 +782,19 @@ class BytesIO(BufferedIOBase):
def getvalue(self): def getvalue(self):
"""Return the bytes value (contents) of the buffer """Return the bytes value (contents) of the buffer
""" """
if self.closed:
raise ValueError("getvalue on closed file")
return bytes(self._buffer) return bytes(self._buffer)
def read(self, n=None): def read(self, n=None):
if self.closed:
raise ValueError("read from closed file")
if n is None: if n is None:
n = -1 n = -1
if n < 0: if n < 0:
n = len(self._buffer) n = len(self._buffer)
if len(self._buffer) <= self._pos:
return self._buffer[:0]
newpos = min(len(self._buffer), self._pos + n) newpos = min(len(self._buffer), self._pos + n)
b = self._buffer[self._pos : newpos] b = self._buffer[self._pos : newpos]
self._pos = newpos self._pos = newpos
...@@ -802,6 +811,8 @@ class BytesIO(BufferedIOBase): ...@@ -802,6 +811,8 @@ class BytesIO(BufferedIOBase):
if isinstance(b, str): if isinstance(b, str):
raise TypeError("can't write str to binary stream") raise TypeError("can't write str to binary stream")
n = len(b) n = len(b)
if n == 0:
return 0
newpos = self._pos + n newpos = self._pos + n
if newpos > len(self._buffer): if newpos > len(self._buffer):
# Inserts null bytes between the current end of the file # Inserts null bytes between the current end of the file
...@@ -813,28 +824,38 @@ class BytesIO(BufferedIOBase): ...@@ -813,28 +824,38 @@ class BytesIO(BufferedIOBase):
return n return n
def seek(self, pos, whence=0): def seek(self, pos, whence=0):
if self.closed:
raise ValueError("seek on closed file")
try: try:
pos = pos.__index__() pos = pos.__index__()
except AttributeError as err: except AttributeError as err:
raise TypeError("an integer is required") from err raise TypeError("an integer is required") from err
if whence == 0: if whence == 0:
self._pos = max(0, pos) self._pos = max(0, pos)
if pos < 0:
raise ValueError("negative seek position %r" % (pos,))
elif whence == 1: elif whence == 1:
self._pos = max(0, self._pos + pos) self._pos = max(0, self._pos + pos)
elif whence == 2: elif whence == 2:
self._pos = max(0, len(self._buffer) + pos) self._pos = max(0, len(self._buffer) + pos)
else: else:
raise IOError("invalid whence value") raise ValueError("invalid whence value")
return self._pos return self._pos
def tell(self): def tell(self):
if self.closed:
raise ValueError("tell on closed file")
return self._pos return self._pos
def truncate(self, pos=None): def truncate(self, pos=None):
if self.closed:
raise ValueError("truncate on closed file")
if pos is None: if pos is None:
pos = self._pos pos = self._pos
elif pos < 0:
raise ValueError("negative truncate position %r" % (pos,))
del self._buffer[pos:] del self._buffer[pos:]
return pos return self.seek(pos)
def readable(self): def readable(self):
return True return True
...@@ -845,6 +866,16 @@ class BytesIO(BufferedIOBase): ...@@ -845,6 +866,16 @@ class BytesIO(BufferedIOBase):
def seekable(self): def seekable(self):
return True return True
# Use the faster implementation of BytesIO if available
try:
import _bytesio
class BytesIO(_bytesio._BytesIO, BufferedIOBase):
__doc__ = _bytesio._BytesIO.__doc__
except ImportError:
BytesIO = _BytesIO
class BufferedReader(_BufferedIOMixin): class BufferedReader(_BufferedIOMixin):
...@@ -978,6 +1009,12 @@ class BufferedWriter(_BufferedIOMixin): ...@@ -978,6 +1009,12 @@ class BufferedWriter(_BufferedIOMixin):
raise BlockingIOError(e.errno, e.strerror, overage) raise BlockingIOError(e.errno, e.strerror, overage)
return written return written
def truncate(self, pos=None):
self.flush()
if pos is None:
pos = self.raw.tell()
return self.raw.truncate(pos)
def flush(self): def flush(self):
if self.closed: if self.closed:
raise ValueError("flush of closed file") raise ValueError("flush of closed file")
...@@ -1097,6 +1134,13 @@ class BufferedRandom(BufferedWriter, BufferedReader): ...@@ -1097,6 +1134,13 @@ class BufferedRandom(BufferedWriter, BufferedReader):
else: else:
return self.raw.tell() - len(self._read_buf) return self.raw.tell() - len(self._read_buf)
def truncate(self, pos=None):
if pos is None:
pos = self.tell()
# Use seek to flush the read buffer.
self.seek(pos)
return BufferedWriter.truncate(self)
def read(self, n=None): def read(self, n=None):
if n is None: if n is None:
n = -1 n = -1
...@@ -1145,11 +1189,7 @@ class TextIOBase(IOBase): ...@@ -1145,11 +1189,7 @@ class TextIOBase(IOBase):
def truncate(self, pos: int = None) -> int: def truncate(self, pos: int = None) -> int:
"""Truncate size to pos.""" """Truncate size to pos."""
self.flush() self._unsupported("truncate")
if pos is None:
pos = self.tell()
self.seek(pos)
return self.buffer.truncate()
def readline(self) -> str: def readline(self) -> str:
"""Read until newline or EOF. """Read until newline or EOF.
...@@ -1346,6 +1386,12 @@ class TextIOWrapper(TextIOBase): ...@@ -1346,6 +1386,12 @@ class TextIOWrapper(TextIOBase):
def seekable(self): def seekable(self):
return self._seekable return self._seekable
def readable(self):
return self.buffer.readable()
def writable(self):
return self.buffer.writable()
def flush(self): def flush(self):
self.buffer.flush() self.buffer.flush()
self._telling = self._seekable self._telling = self._seekable
...@@ -1539,7 +1585,16 @@ class TextIOWrapper(TextIOBase): ...@@ -1539,7 +1585,16 @@ class TextIOWrapper(TextIOBase):
finally: finally:
decoder.setstate(saved_state) decoder.setstate(saved_state)
def truncate(self, pos=None):
self.flush()
if pos is None:
pos = self.tell()
self.seek(pos)
return self.buffer.truncate()
def seek(self, cookie, whence=0): def seek(self, cookie, whence=0):
if self.closed:
raise ValueError("tell on closed file")
if not self._seekable: if not self._seekable:
raise IOError("underlying stream is not seekable") raise IOError("underlying stream is not seekable")
if whence == 1: # seek relative to current position if whence == 1: # seek relative to current position
...@@ -1626,6 +1681,8 @@ class TextIOWrapper(TextIOBase): ...@@ -1626,6 +1681,8 @@ class TextIOWrapper(TextIOBase):
return line return line
def readline(self, limit=None): def readline(self, limit=None):
if self.closed:
raise ValueError("read from closed file")
if limit is None: if limit is None:
limit = -1 limit = -1
......
# Tests StringIO and cStringIO
import sys
import unittest
import io
from test import test_support
class TestGenericStringIO:
# use a class variable CLASS to define which class is being tested
CLASS = None
# Line of data to test as string
_line = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!'
# Constructor to use for the test data (._line is passed to this
# constructor)
constructor = str
def setUp(self):
self._line = self.constructor(self._line)
self._lines = self.constructor((self._line + '\n') * 5)
self._fp = self.CLASS(self._lines)
def test_reads(self):
eq = self.assertEqual
self.assertRaises(TypeError, self._fp.seek)
eq(self._fp.read(10), self._line[:10])
eq(self._fp.readline(), self._line[10:] + '\n')
eq(len(self._fp.readlines(60)), 2)
def test_writes(self):
f = self.CLASS()
self.assertRaises(TypeError, f.seek)
f.write(self._line[:6])
f.seek(3)
f.write(self._line[20:26])
f.write(self._line[52])
self.assertEqual(f.getvalue(), 'abcuvwxyz!')
def test_writelines(self):
f = self.CLASS()
f.writelines([self._line[0], self._line[1], self._line[2]])
f.seek(0)
self.assertEqual(f.getvalue(), 'abc')
def test_writelines_error(self):
def errorGen():
yield 'a'
raise KeyboardInterrupt()
f = self.CLASS()
self.assertRaises(KeyboardInterrupt, f.writelines, errorGen())
def test_truncate(self):
eq = self.assertEqual
f = self.CLASS()
f.write(self._lines)
f.seek(10)
f.truncate()
eq(f.getvalue(), 'abcdefghij')
f.truncate(5)
eq(f.getvalue(), 'abcde')
f.write('xyz')
eq(f.getvalue(), 'abcdexyz')
self.assertRaises(ValueError, f.truncate, -1)
f.close()
self.assertRaises(ValueError, f.write, 'frobnitz')
def test_closed_flag(self):
f = self.CLASS()
self.assertEqual(f.closed, False)
f.close()
self.assertEqual(f.closed, True)
f = self.CLASS(self.constructor("abc"))
self.assertEqual(f.closed, False)
f.close()
self.assertEqual(f.closed, True)
def test_isatty(self):
f = self.CLASS()
self.assertRaises(TypeError, f.isatty, None)
self.assertEqual(f.isatty(), False)
f.close()
self.assertRaises(ValueError, f.isatty)
def test_iterator(self):
eq = self.assertEqual
unless = self.failUnless
eq(iter(self._fp), self._fp)
# Does this object support the iteration protocol?
unless(hasattr(self._fp, '__iter__'))
unless(hasattr(self._fp, '__next__'))
i = 0
for line in self._fp:
eq(line, self._line + '\n')
i += 1
eq(i, 5)
self._fp.close()
self.assertRaises(StopIteration, next, self._fp)
class TestioStringIO(TestGenericStringIO, unittest.TestCase):
CLASS = io.StringIO
def test_unicode(self):
if not test_support.have_unicode: return
# The StringIO module also supports concatenating Unicode
# snippets to larger Unicode strings. This is tested by this
# method. Note that cStringIO does not support this extension.
f = self.CLASS()
f.write(self._line[:6])
f.seek(3)
f.write(str(self._line[20:26]))
f.write(str(self._line[52]))
s = f.getvalue()
self.assertEqual(s, str('abcuvwxyz!'))
self.assertEqual(type(s), str)
def test_main():
test_support.run_unittest(TestioStringIO)
if __name__ == '__main__':
test_main()
...@@ -98,7 +98,7 @@ class IOTest(unittest.TestCase): ...@@ -98,7 +98,7 @@ class IOTest(unittest.TestCase):
self.assertEqual(f.seek(-1, 2), 13) self.assertEqual(f.seek(-1, 2), 13)
self.assertEqual(f.tell(), 13) self.assertEqual(f.tell(), 13)
self.assertEqual(f.truncate(12), 12) self.assertEqual(f.truncate(12), 12)
self.assertEqual(f.tell(), 13) self.assertEqual(f.tell(), 12)
self.assertRaises(TypeError, f.seek, 0.0) self.assertRaises(TypeError, f.seek, 0.0)
def read_ops(self, f, buffered=False): def read_ops(self, f, buffered=False):
...@@ -143,7 +143,7 @@ class IOTest(unittest.TestCase): ...@@ -143,7 +143,7 @@ class IOTest(unittest.TestCase):
self.assertEqual(f.tell(), self.LARGE + 2) self.assertEqual(f.tell(), self.LARGE + 2)
self.assertEqual(f.seek(0, 2), self.LARGE + 2) self.assertEqual(f.seek(0, 2), self.LARGE + 2)
self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1) self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1)
self.assertEqual(f.tell(), self.LARGE + 2) self.assertEqual(f.tell(), self.LARGE + 1)
self.assertEqual(f.seek(0, 2), self.LARGE + 1) self.assertEqual(f.seek(0, 2), self.LARGE + 1)
self.assertEqual(f.seek(-1, 2), self.LARGE) self.assertEqual(f.seek(-1, 2), self.LARGE)
self.assertEqual(f.read(2), b"x") self.assertEqual(f.read(2), b"x")
...@@ -727,6 +727,7 @@ class TextIOWrapperTest(unittest.TestCase): ...@@ -727,6 +727,7 @@ class TextIOWrapperTest(unittest.TestCase):
txt.write("BB\nCCC\n") txt.write("BB\nCCC\n")
txt.write("X\rY\r\nZ") txt.write("X\rY\r\nZ")
txt.flush() txt.flush()
self.assertEquals(buf.closed, False)
self.assertEquals(buf.getvalue(), expected) self.assertEquals(buf.getvalue(), expected)
def testNewlines(self): def testNewlines(self):
...@@ -807,7 +808,8 @@ class TextIOWrapperTest(unittest.TestCase): ...@@ -807,7 +808,8 @@ class TextIOWrapperTest(unittest.TestCase):
txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline) txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline)
txt.write(data) txt.write(data)
txt.close() txt.close()
self.assertEquals(buf.getvalue(), expected) self.assertEquals(buf.closed, True)
self.assertRaises(ValueError, buf.getvalue)
finally: finally:
os.linesep = save_linesep os.linesep = save_linesep
......
...@@ -120,14 +120,15 @@ class TestCase(unittest.TestCase): ...@@ -120,14 +120,15 @@ class TestCase(unittest.TestCase):
newsize -= 1 newsize -= 1
f.seek(42) f.seek(42)
f.truncate(newsize) f.truncate(newsize)
self.assertEqual(f.tell(), 42) # else pointer moved
f.seek(0, 2)
self.assertEqual(f.tell(), newsize) # else wasn't truncated self.assertEqual(f.tell(), newsize) # else wasn't truncated
f.seek(0, 2)
self.assertEqual(f.tell(), newsize)
# XXX truncate(larger than true size) is ill-defined # XXX truncate(larger than true size) is ill-defined
# across platform; cut it waaaaay back # across platform; cut it waaaaay back
f.seek(0) f.seek(0)
f.truncate(1) f.truncate(1)
self.assertEqual(f.tell(), 0) # else pointer moved self.assertEqual(f.tell(), 1) # else pointer moved
f.seek(0)
self.assertEqual(len(f.read()), 1) # else wasn't truncated self.assertEqual(len(f.read()), 1) # else wasn't truncated
def test_main(): def test_main():
......
This diff is collapsed.
...@@ -58,7 +58,7 @@ class MimeToolsTest(unittest.TestCase): ...@@ -58,7 +58,7 @@ class MimeToolsTest(unittest.TestCase):
s.add(nb) s.add(nb)
def test_message(self): def test_message(self):
msg = mimetools.Message(io.StringIO(msgtext1)) msg = mimetools.Message(io.StringIO(str(msgtext1)))
self.assertEqual(msg.gettype(), "text/plain") self.assertEqual(msg.gettype(), "text/plain")
self.assertEqual(msg.getmaintype(), "text") self.assertEqual(msg.getmaintype(), "text")
self.assertEqual(msg.getsubtype(), "plain") self.assertEqual(msg.getsubtype(), "plain")
......
This diff is collapsed.
...@@ -552,11 +552,10 @@ portable_lseek(int fd, PyObject *posobj, int whence) ...@@ -552,11 +552,10 @@ portable_lseek(int fd, PyObject *posobj, int whence)
PyErr_SetString(PyExc_TypeError, "an integer is required"); PyErr_SetString(PyExc_TypeError, "an integer is required");
return NULL; return NULL;
} }
#if !defined(HAVE_LARGEFILE_SUPPORT) #if defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLong(posobj); pos = PyLong_AsLongLong(posobj);
#else #else
pos = PyLong_Check(posobj) ? pos = PyLong_AsLong(posobj);
PyLong_AsLongLong(posobj) : PyLong_AsLong(posobj);
#endif #endif
if (PyErr_Occurred()) if (PyErr_Occurred())
return NULL; return NULL;
...@@ -572,10 +571,10 @@ portable_lseek(int fd, PyObject *posobj, int whence) ...@@ -572,10 +571,10 @@ portable_lseek(int fd, PyObject *posobj, int whence)
if (res < 0) if (res < 0)
return PyErr_SetFromErrno(PyExc_IOError); return PyErr_SetFromErrno(PyExc_IOError);
#if !defined(HAVE_LARGEFILE_SUPPORT) #if defined(HAVE_LARGEFILE_SUPPORT)
return PyLong_FromLong(res);
#else
return PyLong_FromLongLong(res); return PyLong_FromLongLong(res);
#else
return PyLong_FromLong(res);
#endif #endif
} }
...@@ -622,48 +621,29 @@ fileio_truncate(PyFileIOObject *self, PyObject *args) ...@@ -622,48 +621,29 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
return NULL; return NULL;
if (posobj == Py_None || posobj == NULL) { if (posobj == Py_None || posobj == NULL) {
/* Get the current position. */
posobj = portable_lseek(fd, NULL, 1); posobj = portable_lseek(fd, NULL, 1);
if (posobj == NULL) if (posobj == NULL)
return NULL; return NULL;
} }
else { else {
Py_INCREF(posobj); /* Move to the position to be truncated. */
posobj = portable_lseek(fd, posobj, 0);
} }
#if !defined(HAVE_LARGEFILE_SUPPORT) #if defined(HAVE_LARGEFILE_SUPPORT)
pos = PyLong_AsLong(posobj); pos = PyLong_AsLongLong(posobj);
#else #else
pos = PyLong_Check(posobj) ? pos = PyLong_AsLong(posobj);
PyLong_AsLongLong(posobj) : PyLong_AsLong(posobj);
#endif #endif
if (PyErr_Occurred()) { if (PyErr_Occurred())
Py_DECREF(posobj);
return NULL; return NULL;
}
#ifdef MS_WINDOWS #ifdef MS_WINDOWS
/* MS _chsize doesn't work if newsize doesn't fit in 32 bits, /* MS _chsize doesn't work if newsize doesn't fit in 32 bits,
so don't even try using it. */ so don't even try using it. */
{ {
HANDLE hFile; HANDLE hFile;
PyObject *pos2, *oldposobj;
/* store the current position */
oldposobj = portable_lseek(self->fd, NULL, 1);
if (oldposobj == NULL) {
Py_DECREF(posobj);
return NULL;
}
/* Have to move current pos to desired endpoint on Windows. */
errno = 0;
pos2 = portable_lseek(fd, posobj, SEEK_SET);
if (pos2 == NULL) {
Py_DECREF(posobj);
Py_DECREF(oldposobj);
return NULL;
}
Py_DECREF(pos2);
/* Truncate. Note that this may grow the file! */ /* Truncate. Note that this may grow the file! */
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
...@@ -676,18 +656,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args) ...@@ -676,18 +656,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
errno = EACCES; errno = EACCES;
} }
Py_END_ALLOW_THREADS Py_END_ALLOW_THREADS
if (ret == 0) {
/* Move to the previous position in the file */
pos2 = portable_lseek(fd, oldposobj, SEEK_SET);
if (pos2 == NULL) {
Py_DECREF(posobj);
Py_DECREF(oldposobj);
return NULL;
}
}
Py_DECREF(pos2);
Py_DECREF(oldposobj);
} }
#else #else
Py_BEGIN_ALLOW_THREADS Py_BEGIN_ALLOW_THREADS
...@@ -697,7 +665,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args) ...@@ -697,7 +665,6 @@ fileio_truncate(PyFileIOObject *self, PyObject *args)
#endif /* !MS_WINDOWS */ #endif /* !MS_WINDOWS */
if (ret != 0) { if (ret != 0) {
Py_DECREF(posobj);
PyErr_SetFromErrno(PyExc_IOError); PyErr_SetFromErrno(PyExc_IOError);
return NULL; return NULL;
} }
...@@ -791,7 +758,8 @@ PyDoc_STRVAR(seek_doc, ...@@ -791,7 +758,8 @@ PyDoc_STRVAR(seek_doc,
PyDoc_STRVAR(truncate_doc, PyDoc_STRVAR(truncate_doc,
"truncate([size: int]) -> None. Truncate the file to at most size bytes.\n" "truncate([size: int]) -> None. Truncate the file to at most size bytes.\n"
"\n" "\n"
"Size defaults to the current file position, as returned by tell()."); "Size defaults to the current file position, as returned by tell()."
"The current file position is changed to the value of size.");
#endif #endif
PyDoc_STRVAR(tell_doc, PyDoc_STRVAR(tell_doc,
......
...@@ -240,11 +240,11 @@ Py_InitializeEx(int install_sigs) ...@@ -240,11 +240,11 @@ Py_InitializeEx(int install_sigs)
} }
initmain(); /* Module __main__ */ initmain(); /* Module __main__ */
if (!Py_NoSiteFlag)
initsite(); /* Module site */
if (initstdio() < 0) if (initstdio() < 0)
Py_FatalError( Py_FatalError(
"Py_Initialize: can't initialize sys standard streams"); "Py_Initialize: can't initialize sys standard streams");
if (!Py_NoSiteFlag)
initsite(); /* Module site */
/* auto-thread-state API, if available */ /* auto-thread-state API, if available */
#ifdef WITH_THREAD #ifdef WITH_THREAD
......
...@@ -426,6 +426,8 @@ class PyBuildExt(build_ext): ...@@ -426,6 +426,8 @@ class PyBuildExt(build_ext):
exts.append( Extension('operator', ['operator.c']) ) exts.append( Extension('operator', ['operator.c']) )
# _functools # _functools
exts.append( Extension("_functools", ["_functoolsmodule.c"]) ) exts.append( Extension("_functools", ["_functoolsmodule.c"]) )
# Memory-based IO accelerator modules
exts.append( Extension("_bytesio", ["_bytesio.c"]) )
# atexit # atexit
exts.append( Extension("atexit", ["atexitmodule.c"]) ) exts.append( Extension("atexit", ["atexitmodule.c"]) )
# Python C API test module # Python C API test module
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment