Branch merge

Doc/howto/sockets.rst

+.. _socket-howto:
+
 ****************************
   Socket Programming HOWTO
 ****************************

Doc/includes/sqlite3/text_factory.py

@@ -30,14 +30,3 @@ cur.execute("select ?", ("this is latin1 and would normally create errors" +
                          "\xe4\xf6\xfc".encode("latin1"),))
 row = cur.fetchone()
 assert type(row[0]) == str
-
-# sqlite3 offers a built-in optimized text_factory that will return bytestring
-# objects, if the data is in ASCII only, and otherwise return unicode objects
-con.text_factory = sqlite3.OptimizedUnicode
-cur.execute("select ?", (AUSTRIA,))
-row = cur.fetchone()
-assert type(row[0]) == str
-
-cur.execute("select ?", ("Germany",))
-row = cur.fetchone()
-assert type(row[0]) == str

Doc/library/copy.rst

@@ -4,7 +4,11 @@
 .. module:: copy
    :synopsis: Shallow and deep copy operations.
 
-This module provides generic (shallow and deep) copying operations.
+Assignment statements in Python do not copy objects, they create bindings
+between a target and an object. For collections that are mutable or contain
+mutable items, a copy is sometimes needed so one can change one copy without
+changing the other. This module provides generic shallow and deep copy
+operations (explained below).
 
 
 Interface summary:

Doc/library/os.rst

@@ -1411,6 +1411,43 @@ or `the MSDN <http://msdn.microsoft.com/en-us/library/z0kc8e3z.aspx>`_ on Window
    .. versionadded:: 3.3
 
 
+.. _terminal-size:
+
+Querying the size of a terminal
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 3.3
+
+.. function:: get_terminal_size(fd=STDOUT_FILENO)
+
+   Return the size of the terminal window as ``(columns, lines)``,
+   tuple of type :class:`terminal_size`.
+
+   The optional argument ``fd`` (default ``STDOUT_FILENO``, or standard
+   output) specifies which file descriptor should be queried.
+
+   If the file descriptor is not connected to a terminal, an :exc:`OSError`
+   is thrown.
+
+   :func:`shutil.get_terminal_size` is the high-level function which
+   should normally be used, ``os.get_terminal_size`` is the low-level
+   implementation.
+
+   Availability: Unix, Windows.
+
+.. class:: terminal_size(tuple)
+
+   A tuple of ``(columns, lines)`` for holding terminal window size.
+
+   .. attribute:: columns
+
+      Width of the terminal window in characters.
+
+   .. attribute:: lines
+
+      Height of the terminal window in characters.
+
+
 .. _os-file-dir:
 
 Files and Directories

Doc/library/shutil.rst

@@ -459,3 +459,36 @@ The resulting archive contains::
     -rw------- tarek/staff    1675 2008-06-09 13:26:54 ./id_rsa
     -rw-r--r-- tarek/staff     397 2008-06-09 13:26:54 ./id_rsa.pub
     -rw-r--r-- tarek/staff   37192 2010-02-06 18:23:10 ./known_hosts
+
+
+Querying the size of the output terminal
+----------------------------------------
+
+.. versionadded:: 3.3
+
+.. function:: get_terminal_size(fallback=(columns, lines))
+
+   Get the size of the terminal window.
+
+   For each of the two dimensions, the environment variable, ``COLUMNS``
+   and ``LINES`` respectively, is checked. If the variable is defined and
+   the value is a positive integer, it is used.
+
+   When ``COLUMNS`` or ``LINES`` is not defined, which is the common case,
+   the terminal connected to :data:`sys.__stdout__` is queried
+   by invoking :func:`os.get_terminal_size`.
+
+   If the terminal size cannot be successfully queried, either because
+   the system doesn't support querying, or because we are not
+   connected to a terminal, the value given in ``fallback`` parameter
+   is used. ``fallback`` defaults to ``(80, 24)`` which is the default
+   size used by many terminal emulators.
+
+   The value returned is a named tuple of type :class:`os.terminal_size`.
+
+   See also: The Single UNIX Specification, Version 2,
+   `Other Environment Variables`_.
+
+.. _`Other Environment Variables`:
+   http://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html#tag_002_003
+

Doc/library/socket.rst

@@ -981,7 +981,8 @@ correspond to Unix system calls applicable to sockets.
    optional *flags* argument has the same meaning as for :meth:`recv` above.
    Returns the number of bytes sent. Applications are responsible for checking that
    all data has been sent; if only some of the data was transmitted, the
-   application needs to attempt delivery of the remaining data.
+   application needs to attempt delivery of the remaining data. For further
+   information on this topic, consult the :ref:`socket-howto`.
 
 
 .. method:: socket.sendall(bytes[, flags])
@@ -1169,8 +1170,8 @@ using it.  Note that a server must perform the sequence :func:`socket`,
 :meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly
 repeating the :meth:`~socket.accept` to service more than one client), while a
 client only needs the sequence :func:`socket`, :meth:`~socket.connect`.  Also
-note that the server does not :meth:`~socket.send`/:meth:`~socket.recv` on the
-socket it is listening on but on the new socket returned by
+note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on
+the socket it is listening on but on the new socket returned by
 :meth:`~socket.accept`.
 
 The first two examples support IPv4 only. ::
@@ -1188,7 +1189,7 @@ The first two examples support IPv4 only. ::
    while True:
        data = conn.recv(1024)
        if not data: break
-       conn.send(data)
+       conn.sendall(data)
    conn.close()
 
 ::
@@ -1200,7 +1201,7 @@ The first two examples support IPv4 only. ::
    PORT = 50007              # The same port as used by the server
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.connect((HOST, PORT))
-   s.send(b'Hello, world')
+   s.sendall(b'Hello, world')
    data = s.recv(1024)
    s.close()
    print('Received', repr(data))
@@ -1272,7 +1273,7 @@ sends traffic to the first one connected successfully. ::
    if s is None:
        print('could not open socket')
        sys.exit(1)
-   s.send(b'Hello, world')
+   s.sendall(b'Hello, world')
    data = s.recv(1024)
    s.close()
    print('Received', repr(data))

Doc/library/socketserver.rst

@@ -365,7 +365,7 @@ This is the server side::
            print("{} wrote:".format(self.client_address[0]))
            print(self.data)
            # just send back the same data, but upper-cased
-           self.request.send(self.data.upper())
+           self.request.sendall(self.data.upper())
 
    if __name__ == "__main__":
        HOST, PORT = "localhost", 9999
@@ -395,7 +395,7 @@ objects that simplify communication by providing the standard file interface)::
 The difference is that the ``readline()`` call in the second handler will call
 ``recv()`` multiple times until it encounters a newline character, while the
 single ``recv()`` call in the first handler will just return what has been sent
-from the client in one ``send()`` call.
+from the client in one ``sendall()`` call.
 
 
 This is the client side::
@@ -412,7 +412,7 @@ This is the client side::
    try:
        # Connect to server and send data
        sock.connect((HOST, PORT))
-       sock.send(bytes(data + "\n", "utf-8"))
+       sock.sendall(bytes(data + "\n", "utf-8"))
 
        # Receive data from the server and shut down
        received = str(sock.recv(1024), "utf-8")
@@ -510,7 +510,7 @@ An example for the :class:`ThreadingMixIn` class::
            data = str(self.request.recv(1024), 'ascii')
            cur_thread = threading.current_thread()
            response = bytes("{}: {}".format(cur_thread.name, data), 'ascii')
-           self.request.send(response)
+           self.request.sendall(response)
 
    class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
        pass
@@ -519,7 +519,7 @@ An example for the :class:`ThreadingMixIn` class::
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((ip, port))
        try:
-           sock.send(bytes(message, 'ascii'))
+           sock.sendall(bytes(message, 'ascii'))
            response = str(sock.recv(1024), 'ascii')
            print("Received: {}".format(response))
        finally:

Doc/library/sqlite3.rst

@@ -436,10 +436,6 @@ Connection Objects
    :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to
    return bytestrings instead, you can set it to :class:`bytes`.
 
-   For efficiency reasons, there's also a way to return :class:`str` objects
-   only for non-ASCII data, and :class:`bytes` otherwise. To activate it, set
-   this attribute to :const:`sqlite3.OptimizedUnicode`.
-
    You can also set it to any other callable that accepts a single bytestring
    parameter and returns the resulting object.
 

Lib/importlib/__init__.py

@@ -22,9 +22,6 @@ __all__ = ['__import__', 'import_module']
 
 from . import _bootstrap
 
-import os
-import re
-import tokenize
 
 # To simplify imports in test code
 _w_long = _bootstrap._w_long
@@ -32,31 +29,10 @@ _r_long = _bootstrap._r_long
 
 
 # Bootstrap help #####################################################
+import imp
+import sys
 
-# Required built-in modules.
-try:
-    import posix as _os
-except ImportError:
-    try:
-        import nt as _os
-    except ImportError:
-        try:
-            import os2 as _os
-        except ImportError:
-            raise ImportError('posix, nt, or os2 module required for importlib')
-_bootstrap._os = _os
-import imp, sys, marshal, _io
-_bootstrap.imp = imp
-_bootstrap.sys = sys
-_bootstrap.marshal = marshal
-_bootstrap._io = _io
-import _warnings
-_bootstrap._warnings = _warnings
-
-
-from os import sep
-# For os.path.join replacement; pull from Include/osdefs.h:SEP .
-_bootstrap.path_sep = sep
+_bootstrap._setup(sys, imp)
 
 
 # Public API #########################################################

Lib/importlib/_bootstrap.py

@@ -170,7 +170,8 @@ def _write_atomic(path, data):
 def _wrap(new, old):
     """Simple substitute for functools.wraps."""
     for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
-        setattr(new, replace, getattr(old, replace))
+        if hasattr(old, replace):
+            setattr(new, replace, getattr(old, replace))
     new.__dict__.update(old.__dict__)
 
 
@@ -717,7 +718,7 @@ class PathFinder:
         try:
             finder = sys.path_importer_cache[path]
         except KeyError:
-            finder = cls._path_hooks(path)
+            finder = cls._path_hooks(path if path != '' else _os.getcwd())
             sys.path_importer_cache[path] = finder
         else:
             if finder is None and default:
@@ -994,3 +995,46 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0):
                 except ImportError:
                     pass
         return module
+
+
+def _setup(sys_module, imp_module):
+    """Setup importlib by importing needed built-in modules and injecting them
+    into the global namespace.
+
+    As sys is needed for sys.modules access and imp is needed to load built-in
+    modules those two modules must be explicitly passed in.
+
+    """
+    global imp, sys
+    imp = imp_module
+    sys = sys_module
+
+    for module in (imp, sys):
+        if not hasattr(module, '__loader__'):
+            module.__loader__ = BuiltinImporter
+
+    self_module = sys.modules[__name__]
+    for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
+        if builtin_name not in sys.modules:
+            builtin_module = BuiltinImporter.load_module(builtin_name)
+        else:
+            builtin_module = sys.modules[builtin_name]
+        setattr(self_module, builtin_name, builtin_module)
+
+    for builtin_os, path_sep in [('posix', '/'), ('nt', '\\'), ('os2', '\\')]:
+        if builtin_os in sys.modules:
+            os_module = sys.modules[builtin_os]
+            break
+        else:
+            try:
+                os_module = BuiltinImporter.load_module(builtin_os)
+                # TODO: rip out os2 code after 3.3 is released as per PEP 11
+                if builtin_os == 'os2' and 'EMX GCC' in sys.version:
+                    path_sep = '/'
+                break
+            except ImportError:
+                continue
+    else:
+        raise ImportError('importlib requires posix or nt')
+    setattr(self_module, '_os', os_module)
+    setattr(self_module, 'path_sep', path_sep)

Lib/importlib/test/import_/test_path.py

@@ -73,6 +73,16 @@ class FinderTests(unittest.TestCase):
             loader = machinery.PathFinder.find_module(module)
             self.assertTrue(loader is importer)
 
+    def test_path_importer_cache_empty_string(self):
+        # The empty string should create a finder using the cwd.
+        path = ''
+        module = '<test module>'
+        importer = util.mock_modules(module)
+        hook = import_util.mock_path_hook(os.getcwd(), importer=importer)
+        with util.import_state(path=[path], path_hooks=[hook]):
+            loader = machinery.PathFinder.find_module(module)
+            self.assertIs(loader, importer)
+            self.assertIn('', sys.path_importer_cache)
 
 
 class DefaultPathFinderTests(unittest.TestCase):

Lib/multiprocessing/connection.py

@@ -544,7 +544,8 @@ else:
             obsize, ibsize = 0, BUFSIZE
 
         h1 = win32.CreateNamedPipe(
-            address, openmode | win32.FILE_FLAG_OVERLAPPED,
+            address, openmode | win32.FILE_FLAG_OVERLAPPED |
+            win32.FILE_FLAG_FIRST_PIPE_INSTANCE,
             win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
             win32.PIPE_WAIT,
             1, obsize, ibsize, win32.NMPWAIT_WAIT_FOREVER, win32.NULL
@@ -576,7 +577,10 @@ class SocketListener(object):
     def __init__(self, address, family, backlog=1):
         self._socket = socket.socket(getattr(socket, family))
         try:
-            self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            # SO_REUSEADDR has different semantics on Windows (issue #2550).
+            if os.name == 'posix':
+                self._socket.setsockopt(socket.SOL_SOCKET,
+                                        socket.SO_REUSEADDR, 1)
             self._socket.bind(address)
             self._socket.listen(backlog)
             self._address = self._socket.getsockname()
@@ -630,7 +634,8 @@ if sys.platform == 'win32':
         def __init__(self, address, backlog=None):
             self._address = address
             handle = win32.CreateNamedPipe(
-                address, win32.PIPE_ACCESS_DUPLEX,
+                address, win32.PIPE_ACCESS_DUPLEX |
+                win32.FILE_FLAG_FIRST_PIPE_INSTANCE,
                 win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
                 win32.PIPE_WAIT,
                 win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,

Lib/shutil.py

@@ -878,3 +878,46 @@ def chown(path, user=None, group=None):
             raise LookupError("no such group: {!r}".format(group))
 
     os.chown(path, _user, _group)
+
+def get_terminal_size(fallback=(80, 24)):
+    """Get the size of the terminal window.
+
+    For each of the two dimensions, the environment variable, COLUMNS
+    and LINES respectively, is checked. If the variable is defined and
+    the value is a positive integer, it is used.
+
+    When COLUMNS or LINES is not defined, which is the common case,
+    the terminal connected to sys.__stdout__ is queried
+    by invoking os.get_terminal_size.
+
+    If the terminal size cannot be successfully queried, either because
+    the system doesn't support querying, or because we are not
+    connected to a terminal, the value given in fallback parameter
+    is used. Fallback defaults to (80, 24) which is the default
+    size used by many terminal emulators.
+
+    The value returned is a named tuple of type os.terminal_size.
+    """
+    # columns, lines are the working values
+    try:
+        columns = int(os.environ['COLUMNS'])
+    except (KeyError, ValueError):
+        columns = 0
+
+    try:
+        lines = int(os.environ['LINES'])
+    except (KeyError, ValueError):
+        lines = 0
+
+    # only query if necessary
+    if columns <= 0 or lines <= 0:
+        try:
+            size = os.get_terminal_size(sys.__stdout__.fileno())
+        except (NameError, OSError):
+            size = os.terminal_size(fallback)
+        if columns <= 0:
+            columns = size.columns
+        if lines <= 0:
+            lines = size.lines
+
+    return os.terminal_size((columns, lines))

Lib/sqlite3/test/factory.py

@@ -178,6 +178,8 @@ class TextFactoryTests(unittest.TestCase):
         self.assertTrue(row[0].endswith("reich"), "column must contain original data")
 
     def CheckOptimizedUnicode(self):
+        # In py3k, str objects are always returned when text_factory
+        # is OptimizedUnicode
         self.con.text_factory = sqlite.OptimizedUnicode
         austria = "sterreich"
         germany = "Deutchland"

Lib/test/test_multiprocessing.py

@@ -1779,6 +1779,17 @@ class _TestConnection(BaseTestCase):
         self.assertRaises(RuntimeError, reduction.recv_handle, conn)
         p.join()
 
+class _TestListener(BaseTestCase):
+
+    ALLOWED_TYPES = ('processes')
+
+    def test_multiple_bind(self):
+        for family in self.connection.families:
+            l = self.connection.Listener(family=family)
+            self.addCleanup(l.close)
+            self.assertRaises(OSError, self.connection.Listener,
+                              l.address, family)
+
 class _TestListenerClient(BaseTestCase):
 
     ALLOWED_TYPES = ('processes', 'threads')
@@ -1799,6 +1810,7 @@ class _TestListenerClient(BaseTestCase):
             self.assertEqual(conn.recv(), 'hello')
             p.join()
             l.close()
+
 #
 # Test of sending connection and socket objects between processes
 #

Lib/test/test_os.py

@@ -1840,6 +1840,50 @@ class Win32DeprecatedBytesAPI(unittest.TestCase):
                               os.symlink, filename, filename)
 
 
+@unittest.skipUnless(hasattr(os, 'get_terminal_size'), "requires os.get_terminal_size")
+class TermsizeTests(unittest.TestCase):
+    def test_does_not_crash(self):
+        """Check if get_terminal_size() returns a meaningful value.
+
+        There's no easy portable way to actually check the size of the
+        terminal, so let's check if it returns something sensible instead.
+        """
+        try:
+            size = os.get_terminal_size()
+        except OSError as e:
+            if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY):
+                # Under win32 a generic OSError can be thrown if the
+                # handle cannot be retrieved
+                self.skipTest("failed to query terminal size")
+            raise
+
+        self.assertGreaterEqual(size.columns, 0)
+        self.assertGreaterEqual(size.lines, 0)
+
+    def test_stty_match(self):
+        """Check if stty returns the same results
+
+        stty actually tests stdin, so get_terminal_size is invoked on
+        stdin explicitly. If stty succeeded, then get_terminal_size()
+        should work too.
+        """
+        try:
+            size = subprocess.check_output(['stty', 'size']).decode().split()
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            self.skipTest("stty invocation failed")
+        expected = (int(size[1]), int(size[0])) # reversed order
+
+        try:
+            actual = os.get_terminal_size(sys.__stdin__.fileno())
+        except OSError as e:
+            if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY):
+                # Under win32 a generic OSError can be thrown if the
+                # handle cannot be retrieved
+                self.skipTest("failed to query terminal size")
+            raise
+        self.assertEqual(expected, actual)
+
+
 @support.reap_threads
 def test_main():
     support.run_unittest(
@@ -1866,6 +1910,7 @@ def test_main():
         ProgramPriorityTests,
         ExtendedAttributeTests,
         Win32DeprecatedBytesAPI,
+        TermsizeTests,
     )
 
 if __name__ == "__main__":

Lib/test/test_shutil.py

@@ -9,6 +9,7 @@ import os
 import os.path
 import errno
 import functools
+import subprocess
 from test import support
 from test.support import TESTFN
 from os.path import splitdrive
@@ -1267,10 +1268,55 @@ class TestCopyFile(unittest.TestCase):
         finally:
             os.rmdir(dst_dir)
 
+class TermsizeTests(unittest.TestCase):
+    def test_does_not_crash(self):
+        """Check if get_terminal_size() returns a meaningful value.
+
+        There's no easy portable way to actually check the size of the
+        terminal, so let's check if it returns something sensible instead.
+        """
+        size = shutil.get_terminal_size()
+        self.assertGreaterEqual(size.columns, 0)
+        self.assertGreaterEqual(size.lines, 0)
+
+    def test_os_environ_first(self):
+        "Check if environment variables have precedence"
+
+        with support.EnvironmentVarGuard() as env:
+            env['COLUMNS'] = '777'
+            size = shutil.get_terminal_size()
+        self.assertEqual(size.columns, 777)
+
+        with support.EnvironmentVarGuard() as env:
+            env['LINES'] = '888'
+            size = shutil.get_terminal_size()
+        self.assertEqual(size.lines, 888)
+
+    @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty")
+    def test_stty_match(self):
+        """Check if stty returns the same results ignoring env
+
+        This test will fail if stdin and stdout are connected to
+        different terminals with different sizes. Nevertheless, such
+        situations should be pretty rare.
+        """
+        try:
+            size = subprocess.check_output(['stty', 'size']).decode().split()
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            self.skipTest("stty invocation failed")
+        expected = (int(size[1]), int(size[0])) # reversed order
+
+        with support.EnvironmentVarGuard() as env:
+            del env['LINES']
+            del env['COLUMNS']
+            actual = shutil.get_terminal_size()
+
+        self.assertEqual(expected, actual)
 
 
 def test_main():
-    support.run_unittest(TestShutil, TestMove, TestCopyFile)
+    support.run_unittest(TestShutil, TestMove, TestCopyFile,
+                         TermsizeTests)
 
 if __name__ == '__main__':
     test_main()

Lib/test/test_threading.py

@@ -675,6 +675,7 @@ class ThreadJoinOnShutdown(BaseTestCase):
         self.assertFalse(err)
 
     @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()")
+    @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug")
     def test_reinit_tls_after_fork(self):
         # Issue #13817: fork() would deadlock in a multithreaded program with
         # the ad-hoc TLS implementation.

Misc/NEWS

@@ -466,12 +466,33 @@ Core and Builtins
 Library
 -------
 
-- Issue #13845: time.time() now uses GetSystemTimeAsFileTime() instead of
-  ftime() to have a resolution of 100 ns instead of 1 ms (the clock accuracy is
-  between 0.5 ms and 15 ms).
+- Issue #13921: Undocument and clean up sqlite3.OptimizedUnicode,
+  which is obsolete in Python 3.x. It's now aliased to str for
+  backwards compatibility.
+
+- When '' is a path (e.g. in sys.path), make sure __file__ uses the current
+  working directory instead of '' in importlib.
+
+- Issue #13609: Add two functions to query the terminal size:
+  os.get_terminal_size (low level) and shutil.get_terminal_size (high level).
+  Patch by Zbigniew Jędrzejewski-Szmek.
+
+- Issue #13845: On Windows, time.time() now uses GetSystemTimeAsFileTime()
+  instead of ftime() to have a resolution of 100 ns instead of 1 ms (the clock
+  accuracy is between 0.5 ms and 15 ms).
 
 - Issue #13846: Add time.monotonic(), monotonic clock.
 
+- Issue #8184: multiprocessing: On Windows, don't set SO_REUSEADDR on
+  Connection sockets, and set FILE_FLAG_FIRST_PIPE_INSTANCE on named pipes, to
+  make sure two listeners can't bind to the same socket/pipe (or any existing
+  socket/pipe).
+
+- Issue #9021: Add an introduction to the copy module documentation.
+
+- Issue #6005: Examples in the socket library documentation use sendall, where
+  relevant, instead send method.
+
 - Issue #10811: Fix recursive usage of cursors. Instead of crashing,
   raise a ProgrammingError now.
 

Modules/_multiprocessing/win32_functions.c

@@ -784,6 +784,7 @@ create_win32_namespace(void)
     WIN32_CONSTANT(F_DWORD, ERROR_PIPE_BUSY);
     WIN32_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED);
     WIN32_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT);
+    WIN32_CONSTANT(F_DWORD, FILE_FLAG_FIRST_PIPE_INSTANCE);
     WIN32_CONSTANT(F_DWORD, FILE_FLAG_OVERLAPPED);
     WIN32_CONSTANT(F_DWORD, GENERIC_READ);
     WIN32_CONSTANT(F_DWORD, GENERIC_WRITE);

Modules/_sqlite/connection.h

@@ -83,8 +83,7 @@ typedef struct
 
     /* Determines how bytestrings from SQLite are converted to Python objects:
      * - PyUnicode_Type:        Python Unicode objects are constructed from UTF-8 bytestrings
-     * - OptimizedUnicode:      Like before, but for ASCII data, only PyStrings are created.
-     * - PyBytes_Type:         PyStrings are created as-is.
+     * - PyBytes_Type:          The bytestrings are returned as-is.
      * - Any custom callable:   Any object returned from the callable called with the bytestring
      *                          as single parameter.
      */

Modules/_sqlite/cursor.c

@@ -267,11 +267,6 @@ PyObject* _pysqlite_build_column_name(const char* colname)
     }
 }
 
-PyObject* pysqlite_unicode_from_string(const char* val_str, Py_ssize_t size, int optimize)
-{
-    return PyUnicode_FromStringAndSize(val_str, size);
-}
-
 /*
  * Returns a row from the currently active SQLite statement
  *
@@ -355,12 +350,8 @@ PyObject* _pysqlite_fetch_one_row(pysqlite_Cursor* self)
             } else if (coltype == SQLITE_TEXT) {
                 val_str = (const char*)sqlite3_column_text(self->statement->st, i);
                 nbytes = sqlite3_column_bytes(self->statement->st, i);
-                if ((self->connection->text_factory == (PyObject*)&PyUnicode_Type)
-                    || (self->connection->text_factory == pysqlite_OptimizedUnicode)) {
-
-                    converted = pysqlite_unicode_from_string(val_str, nbytes,
-                        self->connection->text_factory == pysqlite_OptimizedUnicode ? 1 : 0);
-
+                if (self->connection->text_factory == (PyObject*)&PyUnicode_Type) {
+                    converted = PyUnicode_FromStringAndSize(val_str, nbytes);
                     if (!converted) {
                         colname = sqlite3_column_name(self->statement->st, i);
                         if (!colname) {
@@ -459,7 +450,6 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
     int statement_type;
     PyObject* descriptor;
     PyObject* second_argument = NULL;
-    int allow_8bit_chars;
 
     if (!check_cursor(self)) {
         goto error;
@@ -468,10 +458,6 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
     self->locked = 1;
     self->reset = 0;
 
-    /* Make shooting yourself in the foot with not utf-8 decodable 8-bit-strings harder */
-    allow_8bit_chars = ((self->connection->text_factory != (PyObject*)&PyUnicode_Type) &&
-        (self->connection->text_factory != pysqlite_OptimizedUnicode));
-
     Py_XDECREF(self->next_row);
     self->next_row = NULL;
 
@@ -630,7 +616,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject*
 
         pysqlite_statement_mark_dirty(self->statement);
 
-        pysqlite_statement_bind_parameters(self->statement, parameters, allow_8bit_chars);
+        pysqlite_statement_bind_parameters(self->statement, parameters);
         if (PyErr_Occurred()) {
             goto error;
         }

Modules/_sqlite/module.c

@@ -37,7 +37,7 @@
 
 PyObject* pysqlite_Error, *pysqlite_Warning, *pysqlite_InterfaceError, *pysqlite_DatabaseError,
     *pysqlite_InternalError, *pysqlite_OperationalError, *pysqlite_ProgrammingError,
-    *pysqlite_IntegrityError, *pysqlite_DataError, *pysqlite_NotSupportedError, *pysqlite_OptimizedUnicode;
+    *pysqlite_IntegrityError, *pysqlite_DataError, *pysqlite_NotSupportedError;
 
 PyObject* converters;
 int _enable_callback_tracebacks;
@@ -407,13 +407,13 @@ PyMODINIT_FUNC PyInit__sqlite3(void)
     }
     PyDict_SetItemString(dict, "NotSupportedError", pysqlite_NotSupportedError);
 
-    /* We just need "something" unique for pysqlite_OptimizedUnicode. It does not really
-     * need to be a string subclass. Just anything that can act as a special
-     * marker for us. So I pulled PyCell_Type out of my magic hat.
-     */
-    Py_INCREF((PyObject*)&PyCell_Type);
-    pysqlite_OptimizedUnicode = (PyObject*)&PyCell_Type;
-    PyDict_SetItemString(dict, "OptimizedUnicode", pysqlite_OptimizedUnicode);
+    /* In Python 2.x, setting Connection.text_factory to
+       OptimizedUnicode caused Unicode objects to be returned for
+       non-ASCII data and bytestrings to be returned for ASCII data.
+       Now OptimizedUnicode is an alias for str, so it has no
+       effect. */
+    Py_INCREF((PyObject*)&PyUnicode_Type);
+    PyDict_SetItemString(dict, "OptimizedUnicode", (PyObject*)&PyUnicode_Type);
 
     /* Set integer constants */
     for (i = 0; _int_constants[i].constant_name != 0; i++) {

Modules/_sqlite/module.h

@@ -38,8 +38,6 @@ extern PyObject* pysqlite_IntegrityError;
 extern PyObject* pysqlite_DataError;
 extern PyObject* pysqlite_NotSupportedError;
 
-extern PyObject* pysqlite_OptimizedUnicode;
-
 /* the functions time.time() and time.sleep() */
 extern PyObject* time_time;
 extern PyObject* time_sleep;

Modules/_sqlite/statement.c

@@ -87,7 +87,7 @@ int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* con
     return rc;
 }
 
-int pysqlite_statement_bind_parameter(pysqlite_Statement* self, int pos, PyObject* parameter, int allow_8bit_chars)
+int pysqlite_statement_bind_parameter(pysqlite_Statement* self, int pos, PyObject* parameter)
 {
     int rc = SQLITE_OK;
     PY_LONG_LONG longlongval;
@@ -166,7 +166,7 @@ static int _need_adapt(PyObject* obj)
     }
 }
 
-void pysqlite_statement_bind_parameters(pysqlite_Statement* self, PyObject* parameters, int allow_8bit_chars)
+void pysqlite_statement_bind_parameters(pysqlite_Statement* self, PyObject* parameters)
 {
     PyObject* current_param;
     PyObject* adapted;
@@ -220,7 +220,7 @@ void pysqlite_statement_bind_parameters(pysqlite_Statement* self, PyObject* para
                 }
             }
 
-            rc = pysqlite_statement_bind_parameter(self, i + 1, adapted, allow_8bit_chars);
+            rc = pysqlite_statement_bind_parameter(self, i + 1, adapted);
             Py_DECREF(adapted);
 
             if (rc != SQLITE_OK) {
@@ -265,7 +265,7 @@ void pysqlite_statement_bind_parameters(pysqlite_Statement* self, PyObject* para
                 }
             }
 
-            rc = pysqlite_statement_bind_parameter(self, i, adapted, allow_8bit_chars);
+            rc = pysqlite_statement_bind_parameter(self, i, adapted);
             Py_DECREF(adapted);
 
             if (rc != SQLITE_OK) {

Modules/_sqlite/statement.h

@@ -46,8 +46,8 @@ extern PyTypeObject pysqlite_StatementType;
 int pysqlite_statement_create(pysqlite_Statement* self, pysqlite_Connection* connection, PyObject* sql);
 void pysqlite_statement_dealloc(pysqlite_Statement* self);
 
-int pysqlite_statement_bind_parameter(pysqlite_Statement* self, int pos, PyObject* parameter, int allow_8bit_chars);
-void pysqlite_statement_bind_parameters(pysqlite_Statement* self, PyObject* parameters, int allow_8bit_chars);
+int pysqlite_statement_bind_parameter(pysqlite_Statement* self, int pos, PyObject* parameter);
+void pysqlite_statement_bind_parameters(pysqlite_Statement* self, PyObject* parameters);
 
 int pysqlite_statement_recompile(pysqlite_Statement* self, PyObject* parameters);
 int pysqlite_statement_finalize(pysqlite_Statement* self);

Modules/posixmodule.c

@@ -125,6 +125,18 @@ corresponding Unix manual entries for more information on calls.");
 #include <dlfcn.h>
 #endif
 
+#if defined(MS_WINDOWS)
+#  define TERMSIZE_USE_CONIO
+#elif defined(HAVE_SYS_IOCTL_H)
+#  include <sys/ioctl.h>
+#  if defined(HAVE_TERMIOS_H)
+#    include <termios.h>
+#  endif
+#  if defined(TIOCGWINSZ)
+#    define TERMSIZE_USE_IOCTL
+#  endif
+#endif /* MS_WINDOWS */
+
 /* Various compilers have only certain posix functions */
 /* XXX Gosh I wish these were all moved into pyconfig.h */
 #if defined(PYCC_VACPP) && defined(PYOS_OS2)
@@ -10477,6 +10489,114 @@ posix_flistxattr(PyObject *self, PyObject *args)
 
 #endif /* USE_XATTRS */
 
+
+/* Terminal size querying */
+
+static PyTypeObject TerminalSizeType;
+
+PyDoc_STRVAR(TerminalSize_docstring,
+    "A tuple of (columns, lines) for holding terminal window size");
+
+static PyStructSequence_Field TerminalSize_fields[] = {
+    {"columns", "width of the terminal window in characters"},
+    {"lines", "height of the terminal window in characters"},
+    {NULL, NULL}
+};
+
+static PyStructSequence_Desc TerminalSize_desc = {
+    "os.terminal_size",
+    TerminalSize_docstring,
+    TerminalSize_fields,
+    2,
+};
+
+#if defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL)
+PyDoc_STRVAR(termsize__doc__,
+    "Return the size of the terminal window as (columns, lines).\n"        \
+    "\n"                                                                   \
+    "The optional argument fd (default standard output) specifies\n"       \
+    "which file descriptor should be queried.\n"                           \
+    "\n"                                                                   \
+    "If the file descriptor is not connected to a terminal, an OSError\n"  \
+    "is thrown.\n"                                                         \
+    "\n"                                                                   \
+    "This function will only be defined if an implementation is\n"         \
+    "available for this system.\n"                                         \
+    "\n"                                                                   \
+    "shutil.get_terminal_size is the high-level function which should \n"  \
+    "normally be used, os.get_terminal_size is the low-level implementation.");
+
+static PyObject*
+get_terminal_size(PyObject *self, PyObject *args)
+{
+    int columns, lines;
+    PyObject *termsize;
+
+    int fd = fileno(stdout);
+    /* Under some conditions stdout may not be connected and
+     * fileno(stdout) may point to an invalid file descriptor. For example
+     * GUI apps don't have valid standard streams by default.
+     *
+     * If this happens, and the optional fd argument is not present,
+     * the ioctl below will fail returning EBADF. This is what we want.
+     */
+
+    if (!PyArg_ParseTuple(args, "|i", &fd))
+        return NULL;
+
+#ifdef TERMSIZE_USE_IOCTL
+    {
+        struct winsize w;
+        if (ioctl(fd, TIOCGWINSZ, &w))
+            return PyErr_SetFromErrno(PyExc_OSError);
+        columns = w.ws_col;
+        lines = w.ws_row;
+    }
+#endif /* TERMSIZE_USE_IOCTL */
+
+#ifdef TERMSIZE_USE_CONIO
+    {
+        DWORD nhandle;
+        HANDLE handle;
+        CONSOLE_SCREEN_BUFFER_INFO csbi;
+        switch (fd) {
+        case 0: nhandle = STD_INPUT_HANDLE;
+            break;
+        case 1: nhandle = STD_OUTPUT_HANDLE;
+            break;
+        case 2: nhandle = STD_ERROR_HANDLE;
+            break;
+        default:
+            return PyErr_Format(PyExc_ValueError, "bad file descriptor");
+        }
+        handle = GetStdHandle(nhandle);
+        if (handle == NULL)
+            return PyErr_Format(PyExc_OSError, "handle cannot be retrieved");
+        if (handle == INVALID_HANDLE_VALUE)
+            return PyErr_SetFromWindowsErr(0);
+
+        if (!GetConsoleScreenBufferInfo(handle, &csbi))
+            return PyErr_SetFromWindowsErr(0);
+
+        columns = csbi.srWindow.Right - csbi.srWindow.Left + 1;
+        lines = csbi.srWindow.Bottom - csbi.srWindow.Top + 1;
+    }
+#endif /* TERMSIZE_USE_CONIO */
+
+    termsize = PyStructSequence_New(&TerminalSizeType);
+    if (termsize == NULL)
+        return NULL;
+    PyStructSequence_SET_ITEM(termsize, 0, PyLong_FromLong(columns));
+    PyStructSequence_SET_ITEM(termsize, 1, PyLong_FromLong(lines));
+    if (PyErr_Occurred()) {
+        Py_DECREF(termsize);
+        return NULL;
+    }
+    return termsize;
+}
+#endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */
+
+
 static PyMethodDef posix_methods[] = {
     {"access",          posix_access, METH_VARARGS, posix_access__doc__},
 #ifdef HAVE_TTYNAME
@@ -10944,6 +11064,9 @@ static PyMethodDef posix_methods[] = {
     {"listxattr", posix_listxattr, METH_VARARGS, posix_listxattr__doc__},
     {"llistxattr", posix_llistxattr, METH_VARARGS, posix_llistxattr__doc__},
     {"flistxattr", posix_flistxattr, METH_VARARGS, posix_flistxattr__doc__},
+#endif
+#if defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL)
+    {"get_terminal_size", get_terminal_size, METH_VARARGS, termsize__doc__},
 #endif
     {NULL,              NULL}            /* Sentinel */
 };
@@ -11539,6 +11662,10 @@ INITFUNC(void)
         PyStructSequence_InitType(&SchedParamType, &sched_param_desc);
         SchedParamType.tp_new = sched_param_new;
 #endif
+
+        /* initialize TerminalSize_info */
+        PyStructSequence_InitType(&TerminalSizeType, &TerminalSize_desc);
+        Py_INCREF(&TerminalSizeType);
     }
 #if defined(HAVE_WAITID) && !defined(__APPLE__)
     Py_INCREF((PyObject*) &WaitidResultType);
@@ -11593,6 +11720,9 @@ INITFUNC(void)
 
 
 #endif /* __APPLE__ */
+
+    PyModule_AddObject(m, "terminal_size", (PyObject*) &TerminalSizeType);
+
     return m;
 
 }

configure

@@ -6144,7 +6144,7 @@ ieeefp.h io.h langinfo.h libintl.h ncurses.h process.h pthread.h \
 sched.h shadow.h signal.h stdint.h stropts.h termios.h \
 unistd.h utime.h \
 poll.h sys/devpoll.h sys/epoll.h sys/poll.h \
-sys/audioio.h sys/xattr.h sys/bsdtty.h sys/event.h sys/file.h \
+sys/audioio.h sys/xattr.h sys/bsdtty.h sys/event.h sys/file.h sys/ioctl.h \
 sys/kern_control.h sys/loadavg.h sys/lock.h sys/mkdev.h sys/modem.h \
 sys/param.h sys/select.h sys/sendfile.h sys/socket.h sys/statvfs.h \
 sys/stat.h sys/syscall.h sys/sys_domain.h sys/termio.h sys/time.h \
@@ -14599,8 +14599,8 @@ esac
 
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 # Files that config.status was made for.
-config_files="$ac_config_files"
-config_headers="$ac_config_headers"
+config_files="`echo $ac_config_files`"
+config_headers="`echo $ac_config_headers`"
 
 _ACEOF
 

configure.in

@@ -1334,7 +1334,7 @@ ieeefp.h io.h langinfo.h libintl.h ncurses.h process.h pthread.h \
 sched.h shadow.h signal.h stdint.h stropts.h termios.h \
 unistd.h utime.h \
 poll.h sys/devpoll.h sys/epoll.h sys/poll.h \
-sys/audioio.h sys/xattr.h sys/bsdtty.h sys/event.h sys/file.h \
+sys/audioio.h sys/xattr.h sys/bsdtty.h sys/event.h sys/file.h sys/ioctl.h \
 sys/kern_control.h sys/loadavg.h sys/lock.h sys/mkdev.h sys/modem.h \
 sys/param.h sys/select.h sys/sendfile.h sys/socket.h sys/statvfs.h \
 sys/stat.h sys/syscall.h sys/sys_domain.h sys/termio.h sys/time.h \

pyconfig.h.in

@@ -908,6 +908,9 @@
 /* Define to 1 if you have the <sys/file.h> header file. */
 #undef HAVE_SYS_FILE_H
 
+/* Define to 1 if you have the <sys/ioctl.h> header file. */
+#undef HAVE_SYS_IOCTL_H
+
 /* Define to 1 if you have the <sys/kern_control.h> header file. */
 #undef HAVE_SYS_KERN_CONTROL_H