Branch merge

Lib/_pyio.py

@@ -556,7 +556,11 @@ class RawIOBase(IOBase):
             if not data:
                 break
             res += data
-        return bytes(res)
+        if res:
+            return bytes(res)
+        else:
+            # b'' or None
+            return data
 
     def readinto(self, b):
         """Read up to len(b) bytes into b.

Lib/socketserver.py

@@ -529,7 +529,6 @@ class ForkingMixIn:
                 self.active_children = []
             self.active_children.append(pid)
             self.close_request(request)
-            return
         else:
             # Child process.
             # This must never return, hence os._exit()!

Lib/test/cjkencodings/hz-utf8.txt

+This sentence is in ASCII.
+The next sentence is in GB.己所不欲，勿施於人。Bye.

Lib/test/cjkencodings/hz.txt

+This sentence is in ASCII.
+The next sentence is in GB.~{<:Ky2;S{#,NpJ)l6HK!#~}Bye.

Lib/test/test_codecencodings_cn.py

@@ -50,6 +50,35 @@ class Test_GB18030(test_multibytecodec_support.TestBase, unittest.TestCase):
     )
     has_iso10646 = True
 
+class Test_HZ(test_multibytecodec_support.TestBase, unittest.TestCase):
+    encoding = 'hz'
+    tstring = test_multibytecodec_support.load_teststring('hz')
+    codectests = (
+        # test '~\n' (3 lines)
+        (b'This sentence is in ASCII.\n'
+         b'The next sentence is in GB.~{<:Ky2;S{#,~}~\n'
+         b'~{NpJ)l6HK!#~}Bye.\n',
+         'strict',
+         'This sentence is in ASCII.\n'
+         'The next sentence is in GB.'
+         '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002'
+         'Bye.\n'),
+        # test '~\n' (4 lines)
+        (b'This sentence is in ASCII.\n'
+         b'The next sentence is in GB.~\n'
+         b'~{<:Ky2;S{#,NpJ)l6HK!#~}~\n'
+         b'Bye.\n',
+         'strict',
+         'This sentence is in ASCII.\n'
+         'The next sentence is in GB.'
+         '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002'
+         'Bye.\n'),
+        # invalid bytes
+        (b'ab~cd', 'replace', 'ab\uFFFDd'),
+        (b'ab\xffcd', 'replace', 'ab\uFFFDcd'),
+        (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'),
+    )
+
 def test_main():
     support.run_unittest(__name__)
 

Lib/test/test_io.py

@@ -790,14 +790,17 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
         # Inject some None's in there to simulate EWOULDBLOCK
         rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None))
         bufio = self.tp(rawio)
-
         self.assertEqual(b"abcd", bufio.read(6))
         self.assertEqual(b"e", bufio.read(1))
         self.assertEqual(b"fg", bufio.read())
         self.assertEqual(b"", bufio.peek(1))
-        self.assertTrue(None is bufio.read())
+        self.assertIsNone(bufio.read())
         self.assertEqual(b"", bufio.read())
 
+        rawio = self.MockRawIO((b"a", None, None))
+        self.assertEqual(b"a", rawio.readall())
+        self.assertIsNone(rawio.readall())
+
     def test_read_past_eof(self):
         rawio = self.MockRawIO((b"abc", b"d", b"efg"))
         bufio = self.tp(rawio)
@@ -2467,6 +2470,8 @@ class MiscIOTest(unittest.TestCase):
             self.assertRaises(ValueError, f.read)
             if hasattr(f, "read1"):
                 self.assertRaises(ValueError, f.read1, 1024)
+            if hasattr(f, "readall"):
+                self.assertRaises(ValueError, f.readall)
             if hasattr(f, "readinto"):
                 self.assertRaises(ValueError, f.readinto, bytearray(1024))
             self.assertRaises(ValueError, f.readline)

Lib/test/test_multibytecodec.py

@@ -257,6 +257,36 @@ class Test_ISO2022(unittest.TestCase):
             # Any ISO 2022 codec will cause the segfault
             myunichr(x).encode('iso_2022_jp', 'ignore')
 
+class TestStateful(unittest.TestCase):
+    text = '\u4E16\u4E16'
+    encoding = 'iso-2022-jp'
+    expected = b'\x1b$B@$@$'
+    expected_reset = b'\x1b$B@$@$\x1b(B'
+
+    def test_encode(self):
+        self.assertEqual(self.text.encode(self.encoding), self.expected_reset)
+
+    def test_incrementalencoder(self):
+        encoder = codecs.getincrementalencoder(self.encoding)()
+        output = b''.join(
+            encoder.encode(char)
+            for char in self.text)
+        self.assertEqual(output, self.expected)
+
+    def test_incrementalencoder_final(self):
+        encoder = codecs.getincrementalencoder(self.encoding)()
+        last_index = len(self.text) - 1
+        output = b''.join(
+            encoder.encode(char, index == last_index)
+            for index, char in enumerate(self.text))
+        self.assertEqual(output, self.expected_reset)
+
+class TestHZStateful(TestStateful):
+    text = '\u804a\u804a'
+    encoding = 'hz'
+    expected = b'~{ADAD'
+    expected_reset = b'~{ADAD~}'
+
 def test_main():
     support.run_unittest(__name__)
 

Makefile.pre.in

@@ -840,8 +840,8 @@ EXTRAPLATDIR= @EXTRAPLATDIR@
 MACHDEPS=	$(PLATDIR) $(EXTRAPLATDIR)
 XMLLIBSUBDIRS=  xml xml/dom xml/etree xml/parsers xml/sax
 LIBSUBDIRS=	tkinter tkinter/test tkinter/test/test_tkinter \
-                tkinter/test/test_ttk site-packages test test/data \
-		test/decimaltestdata \
+		tkinter/test/test_ttk site-packages test test/data \
+		test/cjkencodings test/decimaltestdata \
 		test/tracedmodules \
 		encodings \
 		email email/mime email/test email/test/data \

Misc/NEWS

@@ -75,6 +75,17 @@ Core and Builtins
 Library
 -------
 
+- Issue #12175: RawIOBase.readall() now returns None if read() returns None.
+
+- Issue #12175: FileIO.readall() now raises a ValueError instead of an IOError
+  if the file is closed.
+
+- Issue #12100: Don't reset incremental encoders of CJK codecs at each call to
+  their encode() method anymore, but continue to call the reset() method if the
+  final argument is True.
+
+- Issue #5715: In socketserver, close the server socket in the child process.
+
 - Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore
   to be able to unload the module.
 

Modules/_io/fileio.c

@@ -536,6 +536,8 @@ fileio_readall(fileio *self)
     Py_ssize_t total = 0;
     int n;
 
+    if (self->fd < 0)
+        return err_closed();
     if (!_PyVerify_fd(self->fd))
         return PyErr_SetFromErrno(PyExc_IOError);
 

Modules/_io/iobase.c

@@ -814,6 +814,14 @@ rawiobase_readall(PyObject *self, PyObject *args)
             Py_DECREF(chunks);
             return NULL;
         }
+        if (data == Py_None) {
+            if (PyList_GET_SIZE(chunks) == 0) {
+                Py_DECREF(chunks);
+                return data;
+            }
+            Py_DECREF(data);
+            break;
+        }
         if (!PyBytes_Check(data)) {
             Py_DECREF(chunks);
             Py_DECREF(data);

Modules/cjkcodecs/multibytecodec.c

@@ -479,7 +479,7 @@ multibytecodec_encode(MultibyteCodec *codec,
     MultibyteEncodeBuffer buf;
     Py_ssize_t finalsize, r = 0;
 
-    if (datalen == 0)
+    if (datalen == 0 && !(flags & MBENC_RESET))
         return PyBytes_FromStringAndSize(NULL, 0);
 
     buf.excobj = NULL;
@@ -514,7 +514,7 @@ multibytecodec_encode(MultibyteCodec *codec,
             break;
     }
 
-    if (codec->encreset != NULL)
+    if (codec->encreset != NULL && (flags & MBENC_RESET))
         for (;;) {
             Py_ssize_t outleft;
 
@@ -784,8 +784,8 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
     inbuf_end = inbuf + datalen;
 
     r = multibytecodec_encode(ctx->codec, &ctx->state,
-                    (const Py_UNICODE **)&inbuf,
-                    datalen, ctx->errors, final ? MBENC_FLUSH : 0);
+                    (const Py_UNICODE **)&inbuf, datalen,
+                    ctx->errors, final ? MBENC_FLUSH | MBENC_RESET : 0);
     if (r == NULL) {
         /* recover the original pending buffer */
         if (origpending > 0)

Objects/typeobject.c

@@ -2515,9 +2515,9 @@ static PyMethodDef type_methods[] = {
      PyDoc_STR("__prepare__() -> dict\n"
                "used to create the namespace for the class statement")},
     {"__instancecheck__", type___instancecheck__, METH_O,
-     PyDoc_STR("__instancecheck__() -> check if an object is an instance")},
+     PyDoc_STR("__instancecheck__() -> bool\ncheck if an object is an instance")},
     {"__subclasscheck__", type___subclasscheck__, METH_O,
-     PyDoc_STR("__subclasscheck__() -> check if a class is a subclass")},
+     PyDoc_STR("__subclasscheck__() -> bool\ncheck if a class is a subclass")},
     {0}
 };
 
@@ -3354,7 +3354,7 @@ static PyMethodDef object_methods[] = {
     {"__format__", object_format, METH_VARARGS,
      PyDoc_STR("default object formatter")},
     {"__sizeof__", object_sizeof, METH_NOARGS,
-     PyDoc_STR("__sizeof__() -> size of object in memory, in bytes")},
+     PyDoc_STR("__sizeof__() -> int\nsize of object in memory, in bytes")},
     {0}
 };
 

Tools/msi/msi.py

@@ -1021,6 +1021,8 @@ def add_files(db):
             lib.add_file("zipdir.zip")
         if dir=='tests' and parent.physical=='distutils':
             lib.add_file("Setup.sample")
+        if dir=='cjkencodings':
+            lib.glob("*.txt")
         if dir=='decimaltestdata':
             lib.glob("*.decTest")
         if dir=='output':