Branch merge

Doc/includes/sqlite3/shortcut_methods.py

@@ -17,5 +17,4 @@ con.executemany("insert into person(firstname, lastname) values (?, ?)", persons
 for row in con.execute("select firstname, lastname from person"):
     print(row)
 
-# Using a dummy WHERE clause to not let SQLite take the shortcut table deletes.
-print("I just deleted", con.execute("delete from person where 1=1").rowcount, "rows")
+print("I just deleted", con.execute("delete from person").rowcount, "rows")

Doc/library/sqlite3.rst

@@ -555,18 +555,17 @@ Cursor Objects
    attribute, the database engine's own support for the determination of "rows
    affected"/"rows selected" is quirky.
 
-   For ``DELETE`` statements, SQLite reports :attr:`rowcount` as 0 if you make a
-   ``DELETE FROM table`` without any condition.
-
    For :meth:`executemany` statements, the number of modifications are summed up
    into :attr:`rowcount`.
 
    As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in
    case no ``executeXX()`` has been performed on the cursor or the rowcount of the
-   last operation is not determinable by the interface".
+   last operation is not determinable by the interface". This includes ``SELECT``
+   statements because we cannot determine the number of rows a query produced
+   until all rows were fetched.
 
-   This includes ``SELECT`` statements because we cannot determine the number of
-   rows a query produced until all rows were fetched.
+   With SQLite versions before 3.6.5, :attr:`rowcount` is set to 0 if
+   you make a ``DELETE FROM table`` without any condition.
 
 .. attribute:: Cursor.lastrowid
 

Lib/importlib/test/__main__.py

@@ -7,7 +7,6 @@ builtins.__import__ instead of importlib.__import__.
 from importlib.test.import_ import util
 import os.path
 from test.support import run_unittest
-import sys
 import unittest
 
 
@@ -15,10 +14,17 @@ def test_main():
     start_dir = os.path.dirname(__file__)
     top_dir = os.path.dirname(os.path.dirname(start_dir))
     test_loader = unittest.TestLoader()
-    if '--builtin' in sys.argv:
-        util.using___import__ = True
     run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir))
 
 
 if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser(description='Execute the importlib test '
+                                                  'suite')
+    parser.add_argument('-b', '--builtin', action='store_true', default=False,
+                        help='use builtins.__import__() instead of importlib')
+    args = parser.parse_args()
+    if args.builtin:
+        util.using___import__ = True
     test_main()

Lib/importlib/test/import_/test_api.py

@@ -12,6 +12,13 @@ class APITest(unittest.TestCase):
         with self.assertRaises(TypeError):
             util.import_(42)
 
+    def test_negative_level(self):
+        # Raise ValueError when a negative level is specified.
+        # PEP 328 did away with sys.module None entries and the ambiguity of
+        # absolute/relative imports.
+        with self.assertRaises(ValueError):
+            util.import_('os', globals(), level=-1)
+
 
 def test_main():
     from test.support import run_unittest

Lib/importlib/test/import_/test_path.py

@@ -82,7 +82,7 @@ class FinderTests(unittest.TestCase):
         with util.import_state(path=[path], path_hooks=[hook]):
             loader = machinery.PathFinder.find_module(module)
             self.assertIs(loader, importer)
-            self.assertIn('', sys.path_importer_cache)
+            self.assertIn(os.getcwd(), sys.path_importer_cache)
 
 
 class DefaultPathFinderTests(unittest.TestCase):

Lib/pty.py

@@ -142,15 +142,21 @@ def _copy(master_fd, master_read=_read, stdin_read=_read):
     Copies
             pty master -> standard output   (master_read)
             standard input -> pty master    (stdin_read)"""
-    while 1:
-        rfds, wfds, xfds = select(
-                [master_fd, STDIN_FILENO], [], [])
+    fds = [master_fd, STDIN_FILENO]
+    while True:
+        rfds, wfds, xfds = select(fds, [], [])
         if master_fd in rfds:
             data = master_read(master_fd)
-            os.write(STDOUT_FILENO, data)
+            if not data:  # Reached EOF.
+                fds.remove(master_fd)
+            else:
+                os.write(STDOUT_FILENO, data)
         if STDIN_FILENO in rfds:
             data = stdin_read(STDIN_FILENO)
-            _writen(master_fd, data)
+            if not data:
+                fds.remove(STDIN_FILENO)
+            else:
+                _writen(master_fd, data)
 
 def spawn(argv, master_read=_read, stdin_read=_read):
     """Create a spawned process."""

Lib/test/test_pty.py

@@ -8,7 +8,9 @@ import errno
 import pty
 import os
 import sys
+import select
 import signal
+import socket
 import unittest
 
 TEST_STRING_1 = b"I wish to buy a fish license.\n"
@@ -194,9 +196,96 @@ class PtyTest(unittest.TestCase):
 
         # pty.fork() passed.
 
+
+class SmallPtyTests(unittest.TestCase):
+    """These tests don't spawn children or hang."""
+
+    def setUp(self):
+        self.orig_stdin_fileno = pty.STDIN_FILENO
+        self.orig_stdout_fileno = pty.STDOUT_FILENO
+        self.orig_pty_select = pty.select
+        self.fds = []  # A list of file descriptors to close.
+        self.select_rfds_lengths = []
+        self.select_rfds_results = []
+
+    def tearDown(self):
+        pty.STDIN_FILENO = self.orig_stdin_fileno
+        pty.STDOUT_FILENO = self.orig_stdout_fileno
+        pty.select = self.orig_pty_select
+        for fd in self.fds:
+            try:
+                os.close(fd)
+            except:
+                pass
+
+    def _pipe(self):
+        pipe_fds = os.pipe()
+        self.fds.extend(pipe_fds)
+        return pipe_fds
+
+    def _mock_select(self, rfds, wfds, xfds):
+        # This will raise IndexError when no more expected calls exist.
+        self.assertEqual(self.select_rfds_lengths.pop(0), len(rfds))
+        return self.select_rfds_results.pop(0), [], []
+
+    def test__copy_to_each(self):
+        """Test the normal data case on both master_fd and stdin."""
+        read_from_stdout_fd, mock_stdout_fd = self._pipe()
+        pty.STDOUT_FILENO = mock_stdout_fd
+        mock_stdin_fd, write_to_stdin_fd = self._pipe()
+        pty.STDIN_FILENO = mock_stdin_fd
+        socketpair = socket.socketpair()
+        masters = [s.fileno() for s in socketpair]
+        self.fds.extend(masters)
+
+        # Feed data.  Smaller than PIPEBUF.  These writes will not block.
+        os.write(masters[1], b'from master')
+        os.write(write_to_stdin_fd, b'from stdin')
+
+        # Expect two select calls, the last one will cause IndexError
+        pty.select = self._mock_select
+        self.select_rfds_lengths.append(2)
+        self.select_rfds_results.append([mock_stdin_fd, masters[0]])
+        self.select_rfds_lengths.append(2)
+
+        with self.assertRaises(IndexError):
+            pty._copy(masters[0])
+
+        # Test that the right data went to the right places.
+        rfds = select.select([read_from_stdout_fd, masters[1]], [], [], 0)[0]
+        self.assertEqual([read_from_stdout_fd, masters[1]], rfds)
+        self.assertEqual(os.read(read_from_stdout_fd, 20), b'from master')
+        self.assertEqual(os.read(masters[1], 20), b'from stdin')
+
+    def test__copy_eof_on_all(self):
+        """Test the empty read EOF case on both master_fd and stdin."""
+        read_from_stdout_fd, mock_stdout_fd = self._pipe()
+        pty.STDOUT_FILENO = mock_stdout_fd
+        mock_stdin_fd, write_to_stdin_fd = self._pipe()
+        pty.STDIN_FILENO = mock_stdin_fd
+        socketpair = socket.socketpair()
+        masters = [s.fileno() for s in socketpair]
+        self.fds.extend(masters)
+
+        os.close(masters[1])
+        socketpair[1].close()
+        os.close(write_to_stdin_fd)
+
+        # Expect two select calls, the last one will cause IndexError
+        pty.select = self._mock_select
+        self.select_rfds_lengths.append(2)
+        self.select_rfds_results.append([mock_stdin_fd, masters[0]])
+        # We expect that both fds were removed from the fds list as they
+        # both encountered an EOF before the second select call.
+        self.select_rfds_lengths.append(0)
+
+        with self.assertRaises(IndexError):
+            pty._copy(masters[0])
+
+
 def test_main(verbose=None):
     try:
-        run_unittest(PtyTest)
+        run_unittest(SmallPtyTests, PtyTest)
     finally:
         reap_children()
 

Lib/test/test_sched.py

@@ -12,10 +12,10 @@ class TestCase(unittest.TestCase):
         l = []
         fun = lambda x: l.append(x)
         scheduler = sched.scheduler(time.time, time.sleep)
-        for x in [0.05, 0.04, 0.03, 0.02, 0.01]:
+        for x in [0.5, 0.4, 0.3, 0.2, 0.1]:
             z = scheduler.enter(x, 1, fun, (x,))
         scheduler.run()
-        self.assertEqual(l, [0.01, 0.02, 0.03, 0.04, 0.05])
+        self.assertEqual(l, [0.1, 0.2, 0.3, 0.4, 0.5])
 
     def test_enterabs(self):
         l = []
@@ -31,7 +31,7 @@ class TestCase(unittest.TestCase):
         fun = lambda x: l.append(x)
         scheduler = sched.scheduler(time.time, time.sleep)
         for priority in [1, 2, 3, 4, 5]:
-            z = scheduler.enter(0.01, priority, fun, (priority,))
+            z = scheduler.enterabs(0.01, priority, fun, (priority,))
         scheduler.run()
         self.assertEqual(l, [1, 2, 3, 4, 5])
 
@@ -39,11 +39,12 @@ class TestCase(unittest.TestCase):
         l = []
         fun = lambda x: l.append(x)
         scheduler = sched.scheduler(time.time, time.sleep)
-        event1 = scheduler.enter(0.01, 1, fun, (0.01,))
-        event2 = scheduler.enter(0.02, 1, fun, (0.02,))
-        event3 = scheduler.enter(0.03, 1, fun, (0.03,))
-        event4 = scheduler.enter(0.04, 1, fun, (0.04,))
-        event5 = scheduler.enter(0.05, 1, fun, (0.05,))
+        now = time.time()
+        event1 = scheduler.enterabs(now + 0.01, 1, fun, (0.01,))
+        event2 = scheduler.enterabs(now + 0.02, 1, fun, (0.02,))
+        event3 = scheduler.enterabs(now + 0.03, 1, fun, (0.03,))
+        event4 = scheduler.enterabs(now + 0.04, 1, fun, (0.04,))
+        event5 = scheduler.enterabs(now + 0.05, 1, fun, (0.05,))
         scheduler.cancel(event1)
         scheduler.cancel(event5)
         scheduler.run()
@@ -64,11 +65,12 @@ class TestCase(unittest.TestCase):
         l = []
         fun = lambda x: l.append(x)
         scheduler = sched.scheduler(time.time, time.sleep)
-        e5 = scheduler.enter(0.05, 1, fun)
-        e1 = scheduler.enter(0.01, 1, fun)
-        e2 = scheduler.enter(0.02, 1, fun)
-        e4 = scheduler.enter(0.04, 1, fun)
-        e3 = scheduler.enter(0.03, 1, fun)
+        now = time.time()
+        e5 = scheduler.enterabs(now + 0.05, 1, fun)
+        e1 = scheduler.enterabs(now + 0.01, 1, fun)
+        e2 = scheduler.enterabs(now + 0.02, 1, fun)
+        e4 = scheduler.enterabs(now + 0.04, 1, fun)
+        e3 = scheduler.enterabs(now + 0.03, 1, fun)
         # queue property is supposed to return an order list of
         # upcoming events
         self.assertEqual(list(scheduler.queue), [e1, e2, e3, e4, e5])

Lib/test/test_xml_etree.py

@@ -1352,7 +1352,6 @@ def xinclude():
     r"""
     Basic inclusion example (XInclude C.1)
 
-    >>> from xml.etree import ElementTree as ET
     >>> from xml.etree import ElementInclude
 
     >>> document = xinclude_loader("C1.xml")
@@ -1882,12 +1881,7 @@ class CleanContext(object):
 
     def __enter__(self):
         from xml.etree import ElementPath
-        if hasattr(ET, '_namespace_map'):
-            self._nsmap = ET._namespace_map
-        else:
-            # when testing the cElementTree alias
-            from xml.etree.ElementTree import _namespace_map
-            self._nsmap = _namespace_map
+        self._nsmap = ET.register_namespace._namespace_map
         # Copy the default namespace mapping
         self._nsmap_copy = self._nsmap.copy()
         # Copy the path cache (should be empty)
@@ -1904,12 +1898,20 @@ class CleanContext(object):
         self.checkwarnings.__exit__(*args)
 
 
+class TestAcceleratorNotImported(unittest.TestCase):
+    # Test that the C accelerator was not imported for pyET
+    def test_correct_import_pyET(self):
+        self.assertEqual(pyET.Element.__module__, 'xml.etree.ElementTree')
+
+
 def test_main(module=pyET):
     from test import test_xml_etree
 
     # The same doctests are used for both the Python and the C implementations
     test_xml_etree.ET = module
 
+    support.run_unittest(TestAcceleratorNotImported)
+
     # XXX the C module should give the same warnings as the Python module
     with CleanContext(quiet=(module is not pyET)):
         support.run_doctest(test_xml_etree, verbosity=True)

Lib/test/test_xml_etree_c.py

@@ -5,7 +5,7 @@ from test.support import import_fresh_module
 import unittest
 
 cET = import_fresh_module('xml.etree.ElementTree', fresh=['_elementtree'])
-cET_alias = import_fresh_module('xml.etree.cElementTree', fresh=['_elementtree'])
+cET_alias = import_fresh_module('xml.etree.cElementTree', fresh=['_elementtree', 'xml.etree'])
 
 
 # cElementTree specific tests
@@ -46,6 +46,15 @@ class MiscTests(unittest.TestCase):
         finally:
             data = None
 
+@unittest.skipUnless(cET, 'requires _elementtree')
+class TestAcceleratorImported(unittest.TestCase):
+    # Test that the C accelerator was imported, as expected
+    def test_correct_import_cET(self):
+        self.assertEqual(cET.Element.__module__, '_elementtree')
+
+    def test_correct_import_cET_alias(self):
+        self.assertEqual(cET_alias.Element.__module__, '_elementtree')
+
 
 def test_main():
     from test import test_xml_etree, test_xml_etree_c
@@ -53,7 +62,7 @@ def test_main():
     # Run the tests specific to the C implementation
     support.run_doctest(test_xml_etree_c, verbosity=True)
 
-    support.run_unittest(MiscTests)
+    support.run_unittest(MiscTests, TestAcceleratorImported)
 
     # Run the same test suite as the Python module
     test_xml_etree.test_main(module=cET)

Lib/xml/etree/ElementTree.py

@@ -1086,6 +1086,8 @@ _namespace_map = {
     # dublin core
     "http://purl.org/dc/elements/1.1/": "dc",
 }
+# For tests and troubleshooting
+register_namespace._namespace_map = _namespace_map
 
 def _raise_serialization_error(text):
     raise TypeError(

Makefile.pre.in

@@ -858,7 +858,7 @@ altbininstall: $(BUILDPYTHON)
 	done
 	$(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE)
 	-if test "$(VERSION)" != "$(LDVERSION)"; then \
-		if test -f $(DESTDIR)$(BINDIR)/$(PYTHON)$(VERSION)$(EXE) -o -h $(DESTDIR)$(BINDIR)/$(PYTHON)$(VERSION)$(EXE); \
+		if test -f $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE) -o -h $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \
 		then rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \
 		fi; \
 		(cd $(DESTDIR)$(BINDIR); $(LN) python$(LDVERSION)$(EXE) python$(VERSION)$(EXE)); \
@@ -879,11 +879,11 @@ altbininstall: $(BUILDPYTHON)
 	fi
 
 bininstall: altbininstall
-	-if test -f $(DESTDIR)$(BINDIR)/$(PYTHON)3$(EXE) -o -h $(DESTDIR)$(BINDIR)/$(PYTHON)3$(EXE); \
-	then rm -f $(DESTDIR)$(BINDIR)/$(PYTHON)3$(EXE); \
+	-if test -f $(DESTDIR)$(BINDIR)/python3$(EXE) -o -h $(DESTDIR)$(BINDIR)/python3$(EXE); \
+	then rm -f $(DESTDIR)$(BINDIR)/python3$(EXE); \
 	else true; \
 	fi
-	(cd $(DESTDIR)$(BINDIR); $(LN) python$(VERSION)$(EXE) $(PYTHON)3$(EXE))
+	(cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)$(EXE) python3$(EXE))
 	-if test "$(VERSION)" != "$(LDVERSION)"; then \
 		rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)-config; \
 		(cd $(DESTDIR)$(BINDIR); $(LN) -s python$(LDVERSION)-config python$(VERSION)-config); \

Misc/NEWS

@@ -466,6 +466,13 @@ Core and Builtins
 Library
 -------
 
+- Issue #13961: Move importlib over to using os.replace() for atomic renaming.
+
+- Do away with ambiguous level values (as suggested by PEP 328) in
+  importlib.__import__() by raising ValueError when level < 0.
+
+- Issue #2489: pty.spawn could consume 100% cpu when it encountered an EOF.
+
 - Issue #13014: Fix a possible reference leak in SSLSocket.getpeercert().
 
 - Issue #13777: Add PF_SYSTEM sockets on OS X.
@@ -2256,8 +2263,8 @@ C-API
 Documentation
 -------------
 
-- Issue #13491: Fix many errors in sqlite3 documentation. Initial
-  patch by Johannes Vogel.
+- Issues #13491 and #13995: Fix many errors in sqlite3 documentation.
+  Initial patch for #13491 by Johannes Vogel.
 
 - Issue #13402: Document absoluteness of sys.executable.
 

Modules/_ssl.c

@@ -2547,7 +2547,9 @@ PyInit__ssl(void)
     PyModule_AddIntConstant(m, "OP_CIPHER_SERVER_PREFERENCE",
                             SSL_OP_CIPHER_SERVER_PREFERENCE);
     PyModule_AddIntConstant(m, "OP_SINGLE_DH_USE", SSL_OP_SINGLE_DH_USE);
+#ifdef SSL_OP_SINGLE_ECDH_USE
     PyModule_AddIntConstant(m, "OP_SINGLE_ECDH_USE", SSL_OP_SINGLE_ECDH_USE);
+#endif
 #ifdef SSL_OP_NO_COMPRESSION
     PyModule_AddIntConstant(m, "OP_NO_COMPRESSION",
                             SSL_OP_NO_COMPRESSION);