merge

Doc/library/csv.rst

@@ -164,36 +164,43 @@ The :mod:`csv` module defines the following functions:
 The :mod:`csv` module defines the following classes:
 
 
-.. class:: DictReader(csvfile, fieldnames=None, restkey=None, restval=None, dialect='excel', *args, **kwds)
-
-   Create an object which operates like a regular reader but maps the information
-   read into a dict whose keys are given by the optional  *fieldnames* parameter.
-   If the *fieldnames* parameter is omitted, the values in the first row of the
-   *csvfile* will be used as the fieldnames.  If the row read has more fields
-   than the fieldnames sequence, the remaining data is added as a sequence
-   keyed by the value of *restkey*.  If the row read has fewer fields than the
-   fieldnames sequence, the remaining keys take the value of the optional
-   *restval* parameter.  Any other optional or keyword arguments are passed to
-   the underlying :class:`reader` instance.
-
-
-.. class:: DictWriter(csvfile, fieldnames, restval='', extrasaction='raise', dialect='excel', *args, **kwds)
-
-   Create an object which operates like a regular writer but maps dictionaries onto
-   output rows.  The *fieldnames* parameter identifies the order in which values in
-   the dictionary passed to the :meth:`writerow` method are written to the
-   *csvfile*.  The optional *restval* parameter specifies the value to be written
-   if the dictionary is missing a key in *fieldnames*.  If the dictionary passed to
-   the :meth:`writerow` method contains a key not found in *fieldnames*, the
-   optional *extrasaction* parameter indicates what action to take.  If it is set
-   to ``'raise'`` a :exc:`ValueError` is raised.  If it is set to ``'ignore'``,
-   extra values in the dictionary are ignored.  Any other optional or keyword
-   arguments are passed to the underlying :class:`writer` instance.
-
-   Note that unlike the :class:`DictReader` class, the *fieldnames* parameter of
-   the :class:`DictWriter` is not optional.  Since Python's :class:`dict` objects
-   are not ordered, there is not enough information available to deduce the order
-   in which the row should be written to the *csvfile*.
+.. class:: DictReader(csvfile, fieldnames=None, restkey=None, restval=None, \
+                      dialect='excel', *args, **kwds)
+
+   Create an object which operates like a regular reader but maps the
+   information read into a dict whose keys are given by the optional
+   *fieldnames* parameter.  The *fieldnames* parameter is a :ref:`sequence
+   <collections-abstract-base-classes>` whose elements are associated with the
+   fields of the input data in order. These elements become the keys of the
+   resulting dictionary.  If the *fieldnames* parameter is omitted, the values
+   in the first row of the *csvfile* will be used as the fieldnames.  If the
+   row read has more fields than the fieldnames sequence, the remaining data is
+   added as a sequence keyed by the value of *restkey*.  If the row read has
+   fewer fields than the fieldnames sequence, the remaining keys take the value
+   of the optional *restval* parameter.  Any other optional or keyword
+   arguments are passed to the underlying :class:`reader` instance.
+
+
+.. class:: DictWriter(csvfile, fieldnames, restval='', extrasaction='raise', \
+                      dialect='excel', *args, **kwds)
+
+   Create an object which operates like a regular writer but maps dictionaries
+   onto output rows.  The *fieldnames* parameter is a :ref:`sequence
+   <collections-abstract-base-classes>` of keys that identify the order in
+   which values in the dictionary passed to the :meth:`writerow` method are
+   written to the *csvfile*.  The optional *restval* parameter specifies the
+   value to be written if the dictionary is missing a key in *fieldnames*.  If
+   the dictionary passed to the :meth:`writerow` method contains a key not
+   found in *fieldnames*, the optional *extrasaction* parameter indicates what
+   action to take.  If it is set to ``'raise'`` a :exc:`ValueError` is raised.
+   If it is set to ``'ignore'``, extra values in the dictionary are ignored.
+   Any other optional or keyword arguments are passed to the underlying
+   :class:`writer` instance.
+
+   Note that unlike the :class:`DictReader` class, the *fieldnames* parameter
+   of the :class:`DictWriter` is not optional.  Since Python's :class:`dict`
+   objects are not ordered, there is not enough information available to deduce
+   the order in which the row should be written to the *csvfile*.
 
 
 .. class:: Dialect

Lib/fileinput.py

@@ -387,9 +387,10 @@ def hook_compressed(filename, mode):
 
 
 def hook_encoded(encoding):
-    import codecs
+    import io
     def openhook(filename, mode):
-        return codecs.open(filename, mode, encoding)
+        mode = mode.replace('U', '').replace('b', '') or 'r'
+        return io.open(filename, mode, encoding=encoding, newline='')
     return openhook
 
 

Lib/idlelib/idle_test/README.txt

@@ -41,9 +41,10 @@ idle class. For the benefit of buildbot machines that do not have a graphics
 screen, gui tests must be 'guarded' by "requires('gui')" in a setUp
 function or method. This will typically be setUpClass.
 
-All gui objects must be destroyed by the end of the test, perhaps in a tearDown
-function. Creating the Tk root directly in a setUp allows a reference to be saved
-so it can be properly destroyed in the corresponding tearDown. 
+To avoid interfering with other gui tests, all gui objects must be destroyed
+and deleted by the end of the test.  If a widget, such as a Tk root, is created
+in a setUpX function, destroy it in the corresponding tearDownX.  For module
+and class attributes, also delete the widget.
 ---
     @classmethod
     def setUpClass(cls):
@@ -53,6 +54,7 @@ so it can be properly destroyed in the corresponding tearDown.
     @classmethod
     def tearDownClass(cls):
         cls.root.destroy()
+        del cls.root
 ---
 
 Support.requires('gui') returns true if it is either called in a main module

Lib/idlelib/idle_test/test_formatparagraph.py

@@ -277,6 +277,9 @@ class FormatEventTest(unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         cls.root.destroy()
+        del cls.root
+        del cls.text
+        del cls.formatter
 
     def test_short_line(self):
         self.text.insert('1.0', "Short line\n")

Lib/idlelib/idle_test/test_idlehistory.py

@@ -80,6 +80,7 @@ class FetchTest(unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         cls.root.destroy()
+        del cls.root
 
     def fetch_test(self, reverse, line, prefix, index, bell=False):
         # Perform one fetch as invoked by Alt-N or Alt-P

Lib/idlelib/idle_test/test_searchengine.py

@@ -64,6 +64,7 @@ class GetSelectionTest(unittest.TestCase):
 ##    @classmethod
 ##    def tearDownClass(cls):
 ##        cls.root.destroy()
+##        del cls.root
 
     def test_get_selection(self):
         # text = Text(master=self.root)
@@ -219,6 +220,7 @@ class SearchTest(unittest.TestCase):
 ##    @classmethod
 ##    def tearDownClass(cls):
 ##        cls.root.destroy()
+##        del cls.root
 
     def test_search(self):
         Equal = self.assertEqual
@@ -261,6 +263,7 @@ class ForwardBackwardTest(unittest.TestCase):
 ##    @classmethod
 ##    def tearDownClass(cls):
 ##        cls.root.destroy()
+##        del cls.root
 
     @classmethod
     def setUpClass(cls):

Lib/idlelib/idle_test/test_text.py

@@ -221,6 +221,7 @@ class TkTextTest(TextTest, unittest.TestCase):
     @classmethod
     def tearDownClass(cls):
         cls.root.destroy()
+        del cls.root
 
 
 if __name__ == '__main__':

Lib/lib-tk/turtle.py

@@ -835,7 +835,7 @@ class Shape(object):
             if isinstance(data, list):
                 data = tuple(data)
         elif type_ == "image":
-            if isinstance(data, str):
+            if isinstance(data, basestring):
                 if data.lower().endswith(".gif") and isfile(data):
                     data = TurtleScreen._image(data)
                 # else data assumed to be Photoimage
@@ -1098,7 +1098,7 @@ class TurtleScreen(TurtleScreenBase):
         """
         if len(color) == 1:
             color = color[0]
-        if isinstance(color, str):
+        if isinstance(color, basestring):
             if self._iscolorstring(color) or color == "":
                 return color
             else:
@@ -2602,7 +2602,7 @@ class RawTurtle(TPen, TNavigator):
     def _cc(self, args):
         """Convert colortriples to hexstrings.
         """
-        if isinstance(args, str):
+        if isinstance(args, basestring):
             return args
         try:
             r, g, b = args
@@ -3228,7 +3228,7 @@ class RawTurtle(TPen, TNavigator):
         """
         #print "dot-1:", size, color
         if not color:
-            if isinstance(size, (str, tuple)):
+            if isinstance(size, (basestring, tuple)):
                 color = self._colorstr(size)
                 size = self._pensize + max(self._pensize, 4)
             else:
@@ -3913,7 +3913,7 @@ if __name__ == "__main__":
         down()
         # some text
         write("startstart", 1)
-        write("start", 1)
+        write(u"start", 1)
         color("red")
         # staircase
         for i in range(5):
@@ -3988,7 +3988,7 @@ if __name__ == "__main__":
         tri = getturtle()
         tri.resizemode("auto")
         turtle = Turtle()
-        turtle.resizemode("auto")
+        turtle.resizemode(u"auto")
         turtle.shape("turtle")
         turtle.reset()
         turtle.left(90)
@@ -3998,7 +3998,7 @@ if __name__ == "__main__":
         turtle.lt(30)
         turtle.down()
         turtle.speed(6)
-        turtle.color("blue","orange")
+        turtle.color("blue",u"orange")
         turtle.pensize(2)
         tri.speed(6)
         setheading(towards(turtle))
@@ -4013,9 +4013,9 @@ if __name__ == "__main__":
                 tri.stamp()
                 switchpen()
             count += 1
-        tri.write("CAUGHT! ", font=("Arial", 16, "bold"), align="right")
+        tri.write("CAUGHT! ", font=("Arial", 16, "bold"), align=u"right")
         tri.pencolor("black")
-        tri.pencolor("red")
+        tri.pencolor(u"red")
 
         def baba(xdummy, ydummy):
             clearscreen()

Lib/random.py

@@ -108,7 +108,7 @@ class Random(_random.Random):
 
         if a is None:
             try:
-                a = long(_hexlify(_urandom(16)), 16)
+                a = long(_hexlify(_urandom(32)), 16)
             except NotImplementedError:
                 import time
                 a = long(time.time() * 256) # use fractional seconds

Lib/test/test_fileinput.py

@@ -218,8 +218,49 @@ class FileInputTests(unittest.TestCase):
         finally:
             remove_tempfiles(t1)
 
+    def test_readline(self):
+        with open(TESTFN, 'wb') as f:
+            f.write('A\nB\r\nC\r')
+            # Fill TextIOWrapper buffer.
+            f.write('123456789\n' * 1000)
+            # Issue #20501: readline() shouldn't read whole file.
+            f.write('\x80')
+        self.addCleanup(safe_unlink, TESTFN)
+
+        fi = FileInput(files=TESTFN, openhook=hook_encoded('ascii'), bufsize=8)
+        # The most likely failure is a UnicodeDecodeError due to the entire
+        # file being read when it shouldn't have been.
+        self.assertEqual(fi.readline(), u'A\n')
+        self.assertEqual(fi.readline(), u'B\r\n')
+        self.assertEqual(fi.readline(), u'C\r')
+        with self.assertRaises(UnicodeDecodeError):
+            # Read to the end of file.
+            list(fi)
+        fi.close()
+
+class Test_hook_encoded(unittest.TestCase):
+    """Unit tests for fileinput.hook_encoded()"""
+
+    def test_modes(self):
+        with open(TESTFN, 'wb') as f:
+            # UTF-7 is a convenient, seldom used encoding
+            f.write('A\nB\r\nC\rD+IKw-')
+        self.addCleanup(safe_unlink, TESTFN)
+
+        def check(mode, expected_lines):
+            fi = FileInput(files=TESTFN, mode=mode,
+                           openhook=hook_encoded('utf-7'))
+            lines = list(fi)
+            fi.close()
+            self.assertEqual(lines, expected_lines)
+
+        check('r', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
+        check('rU', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
+        check('U', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
+        check('rb', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
+
 def test_main():
-    run_unittest(BufferSizesTests, FileInputTests)
+    run_unittest(BufferSizesTests, FileInputTests, Test_hook_encoded)
 
 if __name__ == "__main__":
     test_main()

Lib/test/test_idle.py

@@ -14,6 +14,7 @@ if use_resources and 'gui' in use_resources:
     try:
         root = tk.Tk()
         root.destroy()
+        del root
     except tk.TclError:
         while 'gui' in use_resources:
             use_resources.remove('gui')

Lib/test/test_posix.py

@@ -546,7 +546,7 @@ class PosixGroupsTester(unittest.TestCase):
     def test_initgroups(self):
         # find missing group
 
-        g = max(self.saved_groups) + 1
+        g = max(self.saved_groups or [0]) + 1
         name = pwd.getpwuid(posix.getuid()).pw_name
         posix.initgroups(name, g)
         self.assertIn(g, posix.getgroups())

Lib/test/test_re.py

@@ -933,6 +933,19 @@ class ReTests(unittest.TestCase):
         self.assertEqual(out.getvalue().splitlines(),
                          ['literal 102', 'literal 111', 'literal 111'])
 
+    def test_keyword_parameters(self):
+        # Issue #20283: Accepting the string keyword parameter.
+        pat = re.compile(r'(ab)')
+        self.assertEqual(
+            pat.match(string='abracadabra', pos=7, endpos=10).span(), (7, 9))
+        self.assertEqual(
+            pat.search(string='abracadabra', pos=3, endpos=10).span(), (7, 9))
+        self.assertEqual(
+            pat.findall(string='abracadabra', pos=3, endpos=10), ['ab'])
+        self.assertEqual(
+            pat.split(string='abracadabra', maxsplit=1),
+            ['', 'ab', 'racadabra'])
+
 
 def run_re_tests():
     from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR

Misc/ACKS

@@ -33,6 +33,8 @@ Pehr Anderson
 Erik Andersén
 Oliver Andrich
 Ross Andrus
+Juancarlo Añez
+Chris Angelico
 Ankur Ankan
 Heidi Annexstad
 Éric Araujo

Misc/NEWS

@@ -40,6 +40,20 @@ Core and Builtins
 Library
 -------
 
+- Issue #20283: RE pattern methods now accept the string keyword parameters
+  as documented.  The pattern and source keyword parameters are left as
+  deprecated aliases.
+
+- Improve the random module's default seeding to use 256 bits of entropy
+  from os.urandom().  This was already done for Python 3, mildly improving
+  security with a bigger seed space.
+
+- Issue #15618: Make turtle.py compatible with 'from __future__ import
+  unicode_literals'.  Initial patch by Juancarlo Añez.
+
+- Issue #20501: fileinput module no longer reads whole file into memory when using
+  fileinput.hook_encoded.
+
 - Issue #6815: os.path.expandvars() now supports non-ASCII Unicode environment
   variables names and values.
 
@@ -50,7 +64,7 @@ Library
   Based on patch by Stephen Tu.
 
 - Issue #8478: Untokenizer.compat processes first token from iterator input.
-  Patch based on lines from Georg Brandl, Eric Snow, and Gareth Rees.  
+  Patch based on lines from Georg Brandl, Eric Snow, and Gareth Rees.
 
 - Issue #20594: Avoid name clash with the libc function posix_close.
 

Modules/_sre.c

@@ -1875,18 +1875,62 @@ pattern_dealloc(PatternObject* self)
     PyObject_DEL(self);
 }
 
+static int
+check_args_size(const char *name, PyObject* args, PyObject* kw, int n)
+{
+    Py_ssize_t m = PyTuple_GET_SIZE(args) + (kw ? PyDict_Size(kw) : 0);
+    if (m <= n)
+        return 1;
+    PyErr_Format(PyExc_TypeError,
+                 "%s() takes at most %d positional arguments (%zd given)",
+                 name, n, m);
+    return 0;
+}
+
+static PyObject*
+fix_string_param(PyObject *string, PyObject *string2, const char *oldname)
+{
+    if (string2 != NULL) {
+        char buf[100];
+        if (string != NULL) {
+            PyErr_Format(PyExc_TypeError,
+                         "Argument given by name ('%s') and position (1)",
+                         oldname);
+            return NULL;
+        }
+        sprintf(buf, "The '%s' keyword parameter name is deprecated.  "
+                     "Use 'string' instead.", oldname);
+        if (PyErr_Warn(PyExc_DeprecationWarning, buf) < 0)
+            return NULL;
+        return string2;
+    }
+    if (string == NULL) {
+        PyErr_SetString(PyExc_TypeError,
+                        "Required argument 'string' (pos 1) not found");
+        return NULL;
+    }
+    return string;
+}
+
 static PyObject*
 pattern_match(PatternObject* self, PyObject* args, PyObject* kw)
 {
     SRE_STATE state;
     int status;
 
-    PyObject* string;
+    PyObject *string = NULL, *string2 = NULL;
     Py_ssize_t start = 0;
     Py_ssize_t end = PY_SSIZE_T_MAX;
-    static char* kwlist[] = { "pattern", "pos", "endpos", NULL };
-    if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:match", kwlist,
-                                     &string, &start, &end))
+    static char* kwlist[] = { "string", "pos", "endpos", "pattern", NULL };
+    if (!check_args_size("match", args, kw, 3))
+        return NULL;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kw, "|OnnO:match", kwlist,
+                                     &string, &start, &end, &string2))
+        return NULL;
+
+    string = fix_string_param(string, string2, "pattern");
+    if (!string)
         return NULL;
 
     string = state_init(&state, self, string, start, end);
@@ -1920,12 +1964,19 @@ pattern_search(PatternObject* self, PyObject* args, PyObject* kw)
     SRE_STATE state;
     int status;
 
-    PyObject* string;
+    PyObject *string = NULL, *string2 = NULL;
     Py_ssize_t start = 0;
     Py_ssize_t end = PY_SSIZE_T_MAX;
-    static char* kwlist[] = { "pattern", "pos", "endpos", NULL };
-    if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:search", kwlist,
-                                     &string, &start, &end))
+    static char* kwlist[] = { "string", "pos", "endpos", "pattern", NULL };
+    if (!check_args_size("search", args, kw, 3))
+        return NULL;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kw, "|OnnO:search", kwlist,
+                                     &string, &start, &end, &string2))
+        return NULL;
+
+    string = fix_string_param(string, string2, "pattern");
+    if (!string)
         return NULL;
 
     string = state_init(&state, self, string, start, end);
@@ -2055,12 +2106,19 @@ pattern_findall(PatternObject* self, PyObject* args, PyObject* kw)
     int status;
     Py_ssize_t i, b, e;
 
-    PyObject* string;
+    PyObject *string = NULL, *string2 = NULL;
     Py_ssize_t start = 0;
     Py_ssize_t end = PY_SSIZE_T_MAX;
-    static char* kwlist[] = { "source", "pos", "endpos", NULL };
-    if (!PyArg_ParseTupleAndKeywords(args, kw, "O|nn:findall", kwlist,
-                                     &string, &start, &end))
+    static char* kwlist[] = { "string", "pos", "endpos", "source", NULL };
+    if (!check_args_size("findall", args, kw, 3))
+        return NULL;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kw, "|OnnO:findall", kwlist,
+                                     &string, &start, &end, &string2))
+        return NULL;
+
+    string = fix_string_param(string, string2, "source");
+    if (!string)
         return NULL;
 
     string = state_init(&state, self, string, start, end);
@@ -2185,11 +2243,18 @@ pattern_split(PatternObject* self, PyObject* args, PyObject* kw)
     Py_ssize_t i;
     void* last;
 
-    PyObject* string;
+    PyObject *string = NULL, *string2 = NULL;
     Py_ssize_t maxsplit = 0;
-    static char* kwlist[] = { "source", "maxsplit", NULL };
-    if (!PyArg_ParseTupleAndKeywords(args, kw, "O|n:split", kwlist,
-                                     &string, &maxsplit))
+    static char* kwlist[] = { "string", "maxsplit", "source", NULL };
+    if (!check_args_size("split", args, kw, 2))
+        return NULL;
+
+    if (!PyArg_ParseTupleAndKeywords(args, kw, "|OnO:split", kwlist,
+                                     &string, &maxsplit, &string2))
+        return NULL;
+
+    string = fix_string_param(string, string2, "source");
+    if (!string)
         return NULL;
 
     string = state_init(&state, self, string, 0, PY_SSIZE_T_MAX);