Merge follow-up for #11254 and other changes from 3.2

Doc/using/cmdline.rst

@@ -455,7 +455,8 @@ These environment variables influence Python's behavior.
 .. envvar:: PYTHONDONTWRITEBYTECODE
 
    If this is set, Python won't try to write ``.pyc`` or ``.pyo`` files on the
-   import of source modules.
+   import of source modules.  This is equivalent to specifying the :option:`-B`
+   option.
 
 
 .. envvar:: PYTHONIOENCODING

Lib/_pyio.py

@@ -1455,7 +1455,7 @@ class TextIOWrapper(TextIOBase):
     enabled.  With this enabled, on input, the lines endings '\n', '\r',
     or '\r\n' are translated to '\n' before being returned to the
     caller. Conversely, on output, '\n' is translated to the system
-    default line seperator, os.linesep. If newline is any other of its
+    default line separator, os.linesep. If newline is any other of its
     legal values, that newline becomes the newline when the file is read
     and it is returned untranslated. On output, '\n' is converted to the
     newline.

Lib/distutils/command/build_py.py

@@ -2,7 +2,8 @@
 
 Implements the Distutils 'build_py' command."""
 
-import sys, os
+import os
+import imp
 import sys
 from glob import glob
 
@@ -311,9 +312,11 @@ class build_py (Command):
             outputs.append(filename)
             if include_bytecode:
                 if self.compile:
-                    outputs.append(filename + "c")
+                    outputs.append(imp.cache_from_source(filename,
+                                                         debug_override=True))
                 if self.optimize > 0:
-                    outputs.append(filename + "o")
+                    outputs.append(imp.cache_from_source(filename,
+                                                         debug_override=False))
 
         outputs += [
             os.path.join(build_dir, filename)

Lib/distutils/command/install_lib.py

@@ -4,6 +4,7 @@ Implements the Distutils 'install_lib' command
 (install all Python modules)."""
 
 import os
+import imp
 import sys
 
 from distutils.core import Command
@@ -164,9 +165,11 @@ class install_lib(Command):
             if ext != PYTHON_SOURCE_EXTENSION:
                 continue
             if self.compile:
-                bytecode_files.append(py_file + "c")
+                bytecode_files.append(imp.cache_from_source(
+                    py_file, debug_override=True))
             if self.optimize > 0:
-                bytecode_files.append(py_file + "o")
+                bytecode_files.append(imp.cache_from_source(
+                    py_file, debug_override=False))
 
         return bytecode_files
 

Lib/distutils/tests/test_bdist_dumb.py

 """Tests for distutils.command.bdist_dumb."""
 
-import unittest
-import sys
 import os
+import imp
+import sys
+import zipfile
+import unittest
 from test.support import run_unittest
 
 from distutils.core import Distribution
@@ -72,15 +74,24 @@ class BuildDumbTestCase(support.TempdirManager,
 
         # see what we have
         dist_created = os.listdir(os.path.join(pkg_dir, 'dist'))
-        base = "%s.%s" % (dist.get_fullname(), cmd.plat_name)
+        base = "%s.%s.zip" % (dist.get_fullname(), cmd.plat_name)
         if os.name == 'os2':
             base = base.replace(':', '-')
 
-        wanted = ['%s.zip' % base]
-        self.assertEqual(dist_created, wanted)
+        self.assertEqual(dist_created, [base])
 
         # now let's check what we have in the zip file
-        # XXX to be done
+        fp = zipfile.ZipFile(os.path.join('dist', base))
+        try:
+            contents = fp.namelist()
+        finally:
+            fp.close()
+
+        contents = sorted(os.path.basename(fn) for fn in contents)
+        wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2],
+                  'foo.%s.pyc' % imp.get_tag(),
+                  'foo.py']
+        self.assertEqual(contents, sorted(wanted))
 
 def test_suite():
     return unittest.makeSuite(BuildDumbTestCase)

Lib/distutils/tests/test_build_py.py

@@ -2,7 +2,6 @@
 
 import os
 import sys
-import io
 import imp
 import unittest
 
@@ -54,7 +53,6 @@ class BuildPyTestCase(support.TempdirManager,
         # This makes sure the list of outputs includes byte-compiled
         # files for Python modules but not for package data files
         # (there shouldn't *be* byte-code files for those!).
-        #
         self.assertEqual(len(cmd.get_outputs()), 3)
         pkgdest = os.path.join(destination, "pkg")
         files = os.listdir(pkgdest)
@@ -64,15 +62,11 @@ class BuildPyTestCase(support.TempdirManager,
         if sys.dont_write_bytecode:
             self.assertFalse(os.path.exists(pycache_dir))
         else:
-            # XXX even with -O, distutils writes pyc, not pyo; bug?
             pyc_files = os.listdir(pycache_dir)
             self.assertIn("__init__.%s.pyc" % imp.get_tag(), pyc_files)
 
     def test_empty_package_dir(self):
-        # See SF 1668596/1720897.
-        cwd = os.getcwd()
-
-        # create the distribution files.
+        # See bugs #1668596/#1720897
         sources = self.mkdtemp()
         open(os.path.join(sources, "__init__.py"), "w").close()
 
@@ -81,30 +75,55 @@ class BuildPyTestCase(support.TempdirManager,
         open(os.path.join(testdir, "testfile"), "w").close()
 
         os.chdir(sources)
-        old_stdout = sys.stdout
-        sys.stdout = io.StringIO()
+        dist = Distribution({"packages": ["pkg"],
+                             "package_dir": {"pkg": ""},
+                             "package_data": {"pkg": ["doc/*"]}})
+        # script_name need not exist, it just need to be initialized
+        dist.script_name = os.path.join(sources, "setup.py")
+        dist.script_args = ["build"]
+        dist.parse_command_line()
 
         try:
-            dist = Distribution({"packages": ["pkg"],
-                                 "package_dir": {"pkg": ""},
-                                 "package_data": {"pkg": ["doc/*"]}})
-            # script_name need not exist, it just need to be initialized
-            dist.script_name = os.path.join(sources, "setup.py")
-            dist.script_args = ["build"]
-            dist.parse_command_line()
-
-            try:
-                dist.run_commands()
-            except DistutilsFileError:
-                self.fail("failed package_data test when package_dir is ''")
-        finally:
-            # Restore state.
-            os.chdir(cwd)
-            sys.stdout = old_stdout
+            dist.run_commands()
+        except DistutilsFileError:
+            self.fail("failed package_data test when package_dir is ''")
+
+    @unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
+    def test_byte_compile(self):
+        project_dir, dist = self.create_dist(py_modules=['boiledeggs'])
+        os.chdir(project_dir)
+        self.write_file('boiledeggs.py', 'import antigravity')
+        cmd = build_py(dist)
+        cmd.compile = 1
+        cmd.build_lib = 'here'
+        cmd.finalize_options()
+        cmd.run()
+
+        found = os.listdir(cmd.build_lib)
+        self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py'])
+        found = os.listdir(os.path.join(cmd.build_lib, '__pycache__'))
+        self.assertEqual(found, ['boiledeggs.%s.pyc' % imp.get_tag()])
+
+    @unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
+    def test_byte_compile_optimized(self):
+        project_dir, dist = self.create_dist(py_modules=['boiledeggs'])
+        os.chdir(project_dir)
+        self.write_file('boiledeggs.py', 'import antigravity')
+        cmd = build_py(dist)
+        cmd.compile = 0
+        cmd.optimize = 1
+        cmd.build_lib = 'here'
+        cmd.finalize_options()
+        cmd.run()
+
+        found = os.listdir(cmd.build_lib)
+        self.assertEqual(sorted(found), ['__pycache__', 'boiledeggs.py'])
+        found = os.listdir(os.path.join(cmd.build_lib, '__pycache__'))
+        self.assertEqual(sorted(found), ['boiledeggs.%s.pyo' % imp.get_tag()])
 
     def test_dont_write_bytecode(self):
         # makes sure byte_compile is not used
-        pkg_dir, dist = self.create_dist()
+        dist = self.create_dist()[1]
         cmd = build_py(dist)
         cmd.compile = 1
         cmd.optimize = 1
@@ -118,6 +137,7 @@ class BuildPyTestCase(support.TempdirManager,
 
         self.assertIn('byte-compiling is disabled', self.logs[0][1])
 
+
 def test_suite():
     return unittest.makeSuite(BuildPyTestCase)
 

Lib/distutils/tests/test_install.py

 """Tests for distutils.command.install."""
 
 import os
+import imp
 import sys
 import unittest
 import site
@@ -67,10 +68,7 @@ class InstallTestCase(support.TempdirManager,
         check_path(cmd.install_data, destination)
 
     def test_user_site(self):
-        # site.USER_SITE was introduced in 2.6
-        if sys.version < '2.6':
-            return
-
+        # test install with --user
         # preparing the environment for the test
         self.old_user_base = site.USER_BASE
         self.old_user_site = site.USER_SITE
@@ -87,19 +85,17 @@ class InstallTestCase(support.TempdirManager,
         self.old_expand = os.path.expanduser
         os.path.expanduser = _expanduser
 
-        try:
-            # this is the actual test
-            self._test_user_site()
-        finally:
+        def cleanup():
             site.USER_BASE = self.old_user_base
             site.USER_SITE = self.old_user_site
             install_module.USER_BASE = self.old_user_base
             install_module.USER_SITE = self.old_user_site
             os.path.expanduser = self.old_expand
 
-    def _test_user_site(self):
+        self.addCleanup(cleanup)
+
         for key in ('nt_user', 'unix_user', 'os2_home'):
-            self.assertTrue(key in INSTALL_SCHEMES)
+            self.assertIn(key, INSTALL_SCHEMES)
 
         dist = Distribution({'name': 'xx'})
         cmd = install(dist)
@@ -107,14 +103,14 @@ class InstallTestCase(support.TempdirManager,
         # making sure the user option is there
         options = [name for name, short, lable in
                    cmd.user_options]
-        self.assertTrue('user' in options)
+        self.assertIn('user', options)
 
         # setting a value
         cmd.user = 1
 
         # user base and site shouldn't be created yet
-        self.assertTrue(not os.path.exists(self.user_base))
-        self.assertTrue(not os.path.exists(self.user_site))
+        self.assertFalse(os.path.exists(self.user_base))
+        self.assertFalse(os.path.exists(self.user_site))
 
         # let's run finalize
         cmd.ensure_finalized()
@@ -123,8 +119,8 @@ class InstallTestCase(support.TempdirManager,
         self.assertTrue(os.path.exists(self.user_base))
         self.assertTrue(os.path.exists(self.user_site))
 
-        self.assertTrue('userbase' in cmd.config_vars)
-        self.assertTrue('usersite' in cmd.config_vars)
+        self.assertIn('userbase', cmd.config_vars)
+        self.assertIn('usersite', cmd.config_vars)
 
     def test_handle_extra_path(self):
         dist = Distribution({'name': 'xx', 'extra_path': 'path,dirs'})
@@ -177,15 +173,16 @@ class InstallTestCase(support.TempdirManager,
 
     def test_record(self):
         install_dir = self.mkdtemp()
-        project_dir, dist = self.create_dist(scripts=['hello'])
-        self.addCleanup(os.chdir, os.getcwd())
+        project_dir, dist = self.create_dist(py_modules=['hello'],
+                                             scripts=['sayhi'])
         os.chdir(project_dir)
-        self.write_file('hello', "print('o hai')")
+        self.write_file('hello.py', "def main(): print('o hai')")
+        self.write_file('sayhi', 'from hello import main; main()')
 
         cmd = install(dist)
         dist.command_obj['install'] = cmd
         cmd.root = install_dir
-        cmd.record = os.path.join(project_dir, 'RECORD')
+        cmd.record = os.path.join(project_dir, 'filelist')
         cmd.ensure_finalized()
         cmd.run()
 
@@ -196,7 +193,7 @@ class InstallTestCase(support.TempdirManager,
             f.close()
 
         found = [os.path.basename(line) for line in content.splitlines()]
-        expected = ['hello',
+        expected = ['hello.py', 'hello.%s.pyc' % imp.get_tag(), 'sayhi',
                     'UNKNOWN-0.0.0-py%s.%s.egg-info' % sys.version_info[:2]]
         self.assertEqual(found, expected)
 
@@ -204,7 +201,6 @@ class InstallTestCase(support.TempdirManager,
         install_dir = self.mkdtemp()
         project_dir, dist = self.create_dist(ext_modules=[
             Extension('xx', ['xxmodule.c'])])
-        self.addCleanup(os.chdir, os.getcwd())
         os.chdir(project_dir)
         support.copy_xxmodule_c(project_dir)
 
@@ -216,7 +212,7 @@ class InstallTestCase(support.TempdirManager,
         dist.command_obj['install'] = cmd
         dist.command_obj['build_ext'] = buildextcmd
         cmd.root = install_dir
-        cmd.record = os.path.join(project_dir, 'RECORD')
+        cmd.record = os.path.join(project_dir, 'filelist')
         cmd.ensure_finalized()
         cmd.run()
 
@@ -242,6 +238,7 @@ class InstallTestCase(support.TempdirManager,
             install_module.DEBUG = False
         self.assertTrue(len(self.logs) > old_logs_len)
 
+
 def test_suite():
     return unittest.makeSuite(InstallTestCase)
 

Lib/distutils/tests/test_install_lib.py

@@ -10,13 +10,14 @@ from distutils.tests import support
 from distutils.errors import DistutilsOptionError
 from test.support import run_unittest
 
+
 class InstallLibTestCase(support.TempdirManager,
                          support.LoggingSilencer,
                          support.EnvironGuard,
                          unittest.TestCase):
 
     def test_finalize_options(self):
-        pkg_dir, dist = self.create_dist()
+        dist = self.create_dist()[1]
         cmd = install_lib(dist)
 
         cmd.finalize_options()
@@ -35,56 +36,62 @@ class InstallLibTestCase(support.TempdirManager,
 
     @unittest.skipIf(sys.dont_write_bytecode, 'byte-compile disabled')
     def test_byte_compile(self):
-        pkg_dir, dist = self.create_dist()
-        os.chdir(pkg_dir)
+        project_dir, dist = self.create_dist()
+        os.chdir(project_dir)
         cmd = install_lib(dist)
         cmd.compile = cmd.optimize = 1
 
-        f = os.path.join(pkg_dir, 'foo.py')
+        f = os.path.join(project_dir, 'foo.py')
         self.write_file(f, '# python file')
         cmd.byte_compile([f])
-        pyc_file = imp.cache_from_source('foo.py')
+        pyc_file = imp.cache_from_source('foo.py', debug_override=True)
         pyo_file = imp.cache_from_source('foo.py', debug_override=False)
         self.assertTrue(os.path.exists(pyc_file))
         self.assertTrue(os.path.exists(pyo_file))
 
     def test_get_outputs(self):
-        pkg_dir, dist = self.create_dist()
+        project_dir, dist = self.create_dist()
+        os.chdir(project_dir)
+        os.mkdir('spam')
         cmd = install_lib(dist)
 
         # setting up a dist environment
         cmd.compile = cmd.optimize = 1
-        cmd.install_dir = pkg_dir
-        f = os.path.join(pkg_dir, 'foo.py')
-        self.write_file(f, '# python file')
-        cmd.distribution.py_modules = [pkg_dir]
+        cmd.install_dir = self.mkdtemp()
+        f = os.path.join(project_dir, 'spam', '__init__.py')
+        self.write_file(f, '# python package')
         cmd.distribution.ext_modules = [Extension('foo', ['xxx'])]
-        cmd.distribution.packages = [pkg_dir]
+        cmd.distribution.packages = ['spam']
         cmd.distribution.script_name = 'setup.py'
 
-        # get_output should return 4 elements
-        self.assertTrue(len(cmd.get_outputs()) >= 2)
+        # get_outputs should return 4 elements: spam/__init__.py, .pyc and
+        # .pyo, foo.import-tag-abiflags.so / foo.pyd
+        outputs = cmd.get_outputs()
+        self.assertEqual(len(outputs), 4, outputs)
 
     def test_get_inputs(self):
-        pkg_dir, dist = self.create_dist()
+        project_dir, dist = self.create_dist()
+        os.chdir(project_dir)
+        os.mkdir('spam')
         cmd = install_lib(dist)
 
         # setting up a dist environment
         cmd.compile = cmd.optimize = 1
-        cmd.install_dir = pkg_dir
-        f = os.path.join(pkg_dir, 'foo.py')
-        self.write_file(f, '# python file')
-        cmd.distribution.py_modules = [pkg_dir]
+        cmd.install_dir = self.mkdtemp()
+        f = os.path.join(project_dir, 'spam', '__init__.py')
+        self.write_file(f, '# python package')
         cmd.distribution.ext_modules = [Extension('foo', ['xxx'])]
-        cmd.distribution.packages = [pkg_dir]
+        cmd.distribution.packages = ['spam']
         cmd.distribution.script_name = 'setup.py'
 
-        # get_input should return 2 elements
-        self.assertEqual(len(cmd.get_inputs()), 2)
+        # get_inputs should return 2 elements: spam/__init__.py and
+        # foo.import-tag-abiflags.so / foo.pyd
+        inputs = cmd.get_inputs()
+        self.assertEqual(len(inputs), 2, inputs)
 
     def test_dont_write_bytecode(self):
         # makes sure byte_compile is not used
-        pkg_dir, dist = self.create_dist()
+        dist = self.create_dist()[1]
         cmd = install_lib(dist)
         cmd.compile = 1
         cmd.optimize = 1
@@ -98,6 +105,7 @@ class InstallLibTestCase(support.TempdirManager,
 
         self.assertTrue('byte-compiling is disabled' in self.logs[0][1])
 
+
 def test_suite():
     return unittest.makeSuite(InstallLibTestCase)
 

Lib/distutils/tests/test_sdist.py

@@ -288,7 +288,7 @@ class SDistTestCase(PyPIRCCommandTestCase):
     # the following tests make sure there is a nice error message instead
     # of a traceback when parsing an invalid manifest template
 
-    def _test_template(self, content):
+    def _check_template(self, content):
         dist, cmd = self.get_cmd()
         os.chdir(self.tmp_dir)
         self.write_file('MANIFEST.in', content)
@@ -299,17 +299,17 @@ class SDistTestCase(PyPIRCCommandTestCase):
         self.assertEqual(len(warnings), 1)
 
     def test_invalid_template_unknown_command(self):
-        self._test_template('taunt knights *')
+        self._check_template('taunt knights *')
 
     def test_invalid_template_wrong_arguments(self):
         # this manifest command takes one argument
-        self._test_template('prune')
+        self._check_template('prune')
 
     @unittest.skipIf(os.name != 'nt', 'test relevant for Windows only')
     def test_invalid_template_wrong_path(self):
         # on Windows, trailing slashes are not allowed
         # this used to crash instead of raising a warning: #8286
-        self._test_template('include examples/')
+        self._check_template('include examples/')
 
     @unittest.skipUnless(ZLIB_SUPPORT, 'Need zlib support to run')
     def test_get_file_list(self):

Python/import.c

@@ -3908,7 +3908,8 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws)
 }
 
 PyDoc_STRVAR(doc_cache_from_source,
-"Given the path to a .py file, return the path to its .pyc/.pyo file.\n\
+"cache_from_source(path, [debug_override]) -> path\n\
+Given the path to a .py file, return the path to its .pyc/.pyo file.\n\
 \n\
 The .py file does not need to exist; this simply returns the path to the\n\
 .pyc/.pyo file calculated as if the .py file were imported.  The extension\n\
@@ -3940,7 +3941,8 @@ imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws)
 }
 
 PyDoc_STRVAR(doc_source_from_cache,
-"Given the path to a .pyc./.pyo file, return the path to its .py file.\n\
+"source_from_cache(path) -> path\n\
+Given the path to a .pyc./.pyo file, return the path to its .py file.\n\
 \n\
 The .pyc/.pyo file does not need to exist; this simply returns the path to\n\
 the .py file calculated to correspond to the .pyc/.pyo file.  If path\n\