Branch merge: packaging fixes

Doc/library/multiprocessing.rst

@@ -411,6 +411,20 @@ The :mod:`multiprocessing` package mostly replicates the API of the
 
       See :ref:`multiprocessing-auth-keys`.
 
+   .. attribute:: sentinel
+
+      A numeric handle of a system object which will become "ready" when
+      the process ends.
+
+      On Windows, this is an OS handle usable with the ``WaitForSingleObject``
+      and ``WaitForMultipleObjects`` family of API calls.  On Unix, this is
+      a file descriptor usable with primitives from the :mod:`select` module.
+
+      You can use this value if you want to wait on several events at once.
+      Otherwise calling :meth:`join()` is simpler.
+
+      .. versionadded:: 3.3
+
    .. method:: terminate()
 
       Terminate the process.  On Unix this is done using the ``SIGTERM`` signal;

Doc/library/os.rst

@@ -1019,11 +1019,11 @@ as internal buffering of data.
    Availability: Unix, Windows.
 
 
-.. function:: pipe2(flags=0)
+.. function:: pipe2(flags)
 
    Create a pipe with *flags* set atomically.
-   *flags* is optional and can be constructed by ORing together zero or more of
-   these values: :data:`O_NONBLOCK`, :data:`O_CLOEXEC`.
+   *flags* can be constructed by ORing together one or more of these values:
+   :data:`O_NONBLOCK`, :data:`O_CLOEXEC`.
    Return a pair of file descriptors ``(r, w)`` usable for reading and writing,
    respectively.
 

Lib/idlelib/config-main.def

@@ -46,8 +46,8 @@
 [General]
 editor-on-startup= 0
 autosave= 0
-print-command-posix=lpr %s
-print-command-win=start /min notepad /p %s
+print-command-posix=lpr %%s
+print-command-win=start /min notepad /p %%s
 delete-exitfunc= 1
 
 [EditorWindow]

Lib/multiprocessing/forking.py

@@ -101,10 +101,12 @@ else:
 
 if sys.platform != 'win32':
     import time
+    import select
 
     exit = os._exit
     duplicate = os.dup
     close = os.close
+    _select = util._eintr_retry(select.select)
 
     #
     # We define a Popen class similar to the one from subprocess, but
@@ -118,8 +120,12 @@ if sys.platform != 'win32':
             sys.stderr.flush()
             self.returncode = None
 
+            r, w = os.pipe()
+            self.sentinel = r
+
             self.pid = os.fork()
             if self.pid == 0:
+                os.close(r)
                 if 'random' in sys.modules:
                     import random
                     random.seed()
@@ -128,6 +134,11 @@ if sys.platform != 'win32':
                 sys.stderr.flush()
                 os._exit(code)
 
+            # `w` will be closed when the child exits, at which point `r`
+            # will become ready for reading (using e.g. select()).
+            os.close(w)
+            util.Finalize(self, os.close, (r,))
+
         def poll(self, flag=os.WNOHANG):
             if self.returncode is None:
                 try:
@@ -145,20 +156,14 @@ if sys.platform != 'win32':
             return self.returncode
 
         def wait(self, timeout=None):
-            if timeout is None:
-                return self.poll(0)
-            deadline = time.time() + timeout
-            delay = 0.0005
-            while 1:
-                res = self.poll()
-                if res is not None:
-                    break
-                remaining = deadline - time.time()
-                if remaining <= 0:
-                    break
-                delay = min(delay * 2, remaining, 0.05)
-                time.sleep(delay)
-            return res
+            if self.returncode is None:
+                if timeout is not None:
+                    r = _select([self.sentinel], [], [], timeout)[0]
+                    if not r:
+                        return None
+                # This shouldn't block if select() returned successfully.
+                return self.poll(os.WNOHANG if timeout == 0.0 else 0)
+            return self.returncode
 
         def terminate(self):
             if self.returncode is None:
@@ -258,6 +263,7 @@ else:
             self.pid = pid
             self.returncode = None
             self._handle = hp
+            self.sentinel = int(hp)
 
             # send information to child
             prep_data = get_preparation_data(process_obj._name)

Lib/multiprocessing/process.py

@@ -132,6 +132,7 @@ class Process(object):
         else:
             from .forking import Popen
         self._popen = Popen(self)
+        self._sentinel = self._popen.sentinel
         _current_process._children.add(self)
 
     def terminate(self):
@@ -218,6 +219,17 @@ class Process(object):
 
     pid = ident
 
+    @property
+    def sentinel(self):
+        '''
+        Return a file descriptor (Unix) or handle (Windows) suitable for
+        waiting for process termination.
+        '''
+        try:
+            return self._sentinel
+        except AttributeError:
+            raise ValueError("process not started")
+
     def __repr__(self):
         if self is _current_process:
             status = 'started'

Lib/multiprocessing/util.py

@@ -32,9 +32,11 @@
 # SUCH DAMAGE.
 #
 
+import functools
 import itertools
 import weakref
 import atexit
+import select
 import threading        # we want threading to install it's
                         # cleanup function before multiprocessing does
 
@@ -315,3 +317,21 @@ class ForkAwareLocal(threading.local):
         register_after_fork(self, lambda obj : obj.__dict__.clear())
     def __reduce__(self):
         return type(self), ()
+
+
+#
+# Automatic retry after EINTR
+#
+
+def _eintr_retry(func, _errors=(EnvironmentError, select.error)):
+    @functools.wraps(func)
+    def wrapped(*args, **kwargs):
+        while True:
+            try:
+                return func(*args, **kwargs)
+            except _errors as e:
+                # select.error has no `errno` attribute
+                if e.args[0] == errno.EINTR:
+                    continue
+                raise
+    return wrapped

Lib/smtplib.py

@@ -172,27 +172,6 @@ try:
 except ImportError:
     _have_ssl = False
 else:
-    class SSLFakeFile:
-        """A fake file like object that really wraps a SSLObject.
-
-        It only supports what is needed in smtplib.
-        """
-        def __init__(self, sslobj):
-            self.sslobj = sslobj
-
-        def readline(self):
-            str = b""
-            chr = None
-            while chr != b"\n":
-                chr = self.sslobj.read(1)
-                if not chr:
-                    break
-                str += chr
-            return str
-
-        def close(self):
-            pass
-
     _have_ssl = True
 
 
@@ -322,6 +301,7 @@ class SMTP:
         if self.debuglevel > 0:
             print('connect:', (host, port), file=stderr)
         self.sock = self._get_socket(host, port, self.timeout)
+        self.file = None
         (code, msg) = self.getreply()
         if self.debuglevel > 0:
             print("connect:", msg, file=stderr)
@@ -669,7 +649,7 @@ class SMTP:
                 self.sock = context.wrap_socket(self.sock)
             else:
                 self.sock = ssl.wrap_socket(self.sock, keyfile, certfile)
-            self.file = SSLFakeFile(self.sock)
+            self.file = None
             # RFC 3207:
             # The client MUST discard any knowledge obtained from
             # the server, such as the list of SMTP service extensions,
@@ -853,7 +833,6 @@ if _have_ssl:
                 new_socket = self.context.wrap_socket(new_socket)
             else:
                 new_socket = ssl.wrap_socket(new_socket, self.keyfile, self.certfile)
-            self.file = SSLFakeFile(new_socket)
             return new_socket
 
     __all__.append("SMTP_SSL")
@@ -890,6 +869,7 @@ class LMTP(SMTP):
         # Handle Unix-domain sockets.
         try:
             self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            self.file = None
             self.sock.connect(host)
         except socket.error as msg:
             if self.debuglevel > 0:

Lib/test/support.py

@@ -1561,9 +1561,10 @@ def can_symlink():
     try:
         os.symlink(TESTFN, symlink_path)
         can = True
-        os.remove(symlink_path)
     except (OSError, NotImplementedError):
         can = False
+    else:
+        os.remove(symlink_path)
     _can_symlink = can
     return can
 

Lib/test/test_multiprocessing.py

@@ -71,6 +71,23 @@ HAVE_GETVALUE = not getattr(_multiprocessing,
                             'HAVE_BROKEN_SEM_GETVALUE', False)
 
 WIN32 = (sys.platform == "win32")
+if WIN32:
+    from _subprocess import WaitForSingleObject, INFINITE, WAIT_OBJECT_0
+
+    def wait_for_handle(handle, timeout):
+        if timeout is None or timeout < 0.0:
+            timeout = INFINITE
+        else:
+            timeout = int(1000 * timeout)
+        return WaitForSingleObject(handle, timeout) == WAIT_OBJECT_0
+else:
+    from select import select
+    _select = util._eintr_retry(select)
+
+    def wait_for_handle(handle, timeout):
+        if timeout is not None and timeout < 0.0:
+            timeout = None
+        return handle in _select([handle], [], [], timeout)[0]
 
 #
 # Some tests require ctypes
@@ -307,6 +324,26 @@ class _TestProcess(BaseTestCase):
             ]
         self.assertEqual(result, expected)
 
+    @classmethod
+    def _test_sentinel(cls, event):
+        event.wait(10.0)
+
+    def test_sentinel(self):
+        if self.TYPE == "threads":
+            return
+        event = self.Event()
+        p = self.Process(target=self._test_sentinel, args=(event,))
+        with self.assertRaises(ValueError):
+            p.sentinel
+        p.start()
+        self.addCleanup(p.join)
+        sentinel = p.sentinel
+        self.assertIsInstance(sentinel, int)
+        self.assertFalse(wait_for_handle(sentinel, timeout=0.0))
+        event.set()
+        p.join()
+        self.assertTrue(wait_for_handle(sentinel, timeout=DELTA))
+
 #
 #
 #

Lib/test/test_posix.py

@@ -481,8 +481,8 @@ class PosixTester(unittest.TestCase):
         self.assertRaises(TypeError, os.pipe2, 'DEADBEEF')
         self.assertRaises(TypeError, os.pipe2, 0, 0)
 
-        # try calling without flag, like os.pipe()
-        r, w = os.pipe2()
+        # try calling with flags = 0, like os.pipe()
+        r, w = os.pipe2(0)
         os.close(r)
         os.close(w)
 

Lib/test/test_subprocess.py

@@ -127,9 +127,10 @@ class ProcessTestCase(BaseTestCase):
         with self.assertRaises(subprocess.TimeoutExpired) as c:
             output = subprocess.check_output(
                     [sys.executable, "-c",
-                     "import sys; sys.stdout.write('BDFL')\n"
+                     "import sys, time\n"
+                     "sys.stdout.write('BDFL')\n"
                      "sys.stdout.flush()\n"
-                     "while True: pass"],
+                     "time.sleep(3600)"],
                     # Some heavily loaded buildbots (sparc Debian 3.x) require
                     # this much time to start and print.
                     timeout=3)

Misc/NEWS

@@ -187,6 +187,13 @@ Core and Builtins
 Library
 -------
 
+- Issue #12040: Expose a new attribute ``sentinel`` on instances of
+  :class:`multiprocessing.Process`.  Also, fix Process.join() to not use
+  polling anymore, when given a timeout.
+
+- Issue #11893: Remove obsolete internal wrapper class ``SSLFakeFile`` in the
+  smtplib module.  Patch by Catalin Iacob.
+
 - Issue #12080: Fix a Decimal.power() case that took an unreasonably long time
   to compute.
 
@@ -764,6 +771,9 @@ Library
 Build
 -----
 
+- Do not accidentally include the directory containing sqlite.h twice when
+  building sqlite3.
+
 - Issue #11217: For 64-bit/32-bit Mac OS X universal framework builds,
   ensure "make install" creates symlinks in --prefix bin for the "-32"
   files in the framework bin directory like the installer does.

Modules/arraymodule.c

@@ -2091,7 +2091,7 @@ array_repr(arrayobject *a)
     if (len == 0) {
         return PyUnicode_FromFormat("array('%c')", (int)typecode);
     }
-    if ((typecode == 'u'))
+    if ('u' == typecode)
         v = array_tounicode(a, NULL);
     else
         v = array_tolist(a, NULL);

Modules/mmapmodule.c

@@ -645,9 +645,9 @@ mmap_move_method(mmap_object *self, PyObject *args)
         return NULL;
     } else {
         /* bounds check the values */
-        if (cnt < 0 || (cnt + dest) < cnt || (cnt + src) < cnt ||
-           src < 0 || src > self->size || (src + cnt) > self->size ||
-           dest < 0 || dest > self->size || (dest + cnt) > self->size) {
+        if ((cnt + dest) < cnt || (cnt + src) < cnt ||
+           src > self->size || (src + cnt) > self->size ||
+           dest > self->size || (dest + cnt) > self->size) {
             PyErr_SetString(PyExc_ValueError,
                 "source, destination, or count out of range");
             return NULL;

Modules/nismodule.c

@@ -411,7 +411,7 @@ nis_maps (PyObject *self, PyObject *args, PyObject *kwdict)
         return NULL;
     if ((list = PyList_New(0)) == NULL)
         return NULL;
-    for (maps = maps; maps; maps = maps->next) {
+    for (; maps; maps = maps->next) {
         PyObject *str = PyUnicode_FromString(maps->map);
         if (!str || PyList_Append(list, str) < 0)
         {

Modules/posixmodule.c

@@ -6549,20 +6549,21 @@ posix_pipe(PyObject *self, PyObject *noargs)
 
 #ifdef HAVE_PIPE2
 PyDoc_STRVAR(posix_pipe2__doc__,
-"pipe2(flags=0) -> (read_end, write_end)\n\n\
-Create a pipe with flags set atomically.\
-flags is optional and can be constructed by ORing together zero or more\n\
-of these values: O_NONBLOCK, O_CLOEXEC.\n\
+"pipe2(flags) -> (read_end, write_end)\n\n\
+Create a pipe with flags set atomically.\n\
+flags can be constructed by ORing together one or more of these values:\n\
+O_NONBLOCK, O_CLOEXEC.\n\
 ");
 
 static PyObject *
-posix_pipe2(PyObject *self, PyObject *args)
+posix_pipe2(PyObject *self, PyObject *arg)
 {
-    int flags = 0;
+    int flags;
     int fds[2];
     int res;
 
-    if (!PyArg_ParseTuple(args, "|i:pipe2", &flags))
+    flags = PyLong_AsLong(arg);
+    if (flags == -1 && PyErr_Occurred())
         return NULL;
 
     res = pipe2(fds, flags);
@@ -9467,7 +9468,7 @@ static PyMethodDef posix_methods[] = {
     {"pipe",            posix_pipe, METH_NOARGS, posix_pipe__doc__},
 #endif
 #ifdef HAVE_PIPE2
-    {"pipe2",           posix_pipe2, METH_VARARGS, posix_pipe2__doc__},
+    {"pipe2",           posix_pipe2, METH_O, posix_pipe2__doc__},
 #endif
 #ifdef HAVE_MKFIFO
     {"mkfifo",          posix_mkfifo, METH_VARARGS, posix_mkfifo__doc__},

setup.py

@@ -1045,10 +1045,15 @@ class PyBuildExt(build_ext):
             else:
                 sqlite_extra_link_args = ()
 
+            include_dirs = ["Modules/_sqlite"]
+            # Only include the directory where sqlite was found if it does
+            # not already exist in set include directories, otherwise you
+            # can end up with a bad search path order.
+            if sqlite_incdir not in self.compiler.include_dirs:
+                include_dirs.append(sqlite_incdir)
             exts.append(Extension('_sqlite3', sqlite_srcs,
                                   define_macros=sqlite_defines,
-                                  include_dirs=["Modules/_sqlite",
-                                                sqlite_incdir],
+                                  include_dirs=include_dirs,
                                   library_dirs=sqlite_libdir,
                                   runtime_library_dirs=sqlite_libdir,
                                   extra_link_args=sqlite_extra_link_args,