merge heads

Doc/library/functions.rst

@@ -886,8 +886,8 @@ are always available.  They are listed here in alphabetical order.
    *buffering* is an optional integer used to set the buffering policy.  Pass 0
    to switch buffering off (only allowed in binary mode), 1 to select line
    buffering (only usable in text mode), and an integer > 1 to indicate the size
-   of a fixed-size chunk buffer.  When no *buffering* argument is given, the
-   default buffering policy works as follows:
+   in bytes of a fixed-size chunk buffer.  When no *buffering* argument is
+   given, the default buffering policy works as follows:
 
    * Binary files are buffered in fixed-size chunks; the size of the buffer is
      chosen using a heuristic trying to determine the underlying device's "block

Doc/library/http.server.rst

@@ -181,7 +181,7 @@ of which this module provides three different variants:
 
       .. versionchanged:: 3.4
          The error response includes a Content-Length header.
-         explain argument was added.
+         Added the *explain* argument.
 
 
    .. method:: send_response(code, message=None)

Lib/distutils/tests/test_bdist_dumb.py

@@ -86,9 +86,9 @@ class BuildDumbTestCase(support.TempdirManager,
             fp.close()
 
         contents = sorted(os.path.basename(fn) for fn in contents)
-        wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2],
-                  'foo.%s.pyc' % imp.get_tag(),
-                  'foo.py']
+        wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py']
+        if not sys.dont_write_bytecode:
+            wanted.append('foo.%s.pyc' % imp.get_tag())
         self.assertEqual(contents, sorted(wanted))
 
 def test_suite():

Lib/re.py

@@ -215,8 +215,8 @@ def compile(pattern, flags=0):
 
 def purge():
     "Clear the regular expression caches"
-    _compile.cache_clear()
-    _compile_repl.cache_clear()
+    _cache.clear()
+    _cache_repl.clear()
 
 def template(pattern, flags=0):
     "Compile a template pattern, returning a pattern object"
@@ -259,11 +259,18 @@ def escape(pattern):
 # --------------------------------------------------------------------
 # internals
 
+_cache = {}
+_cache_repl = {}
+
 _pattern_type = type(sre_compile.compile("", 0))
 
-@functools.lru_cache(maxsize=512, typed=True)
+_MAXCACHE = 512
 def _compile(pattern, flags):
     # internal: compile pattern
+    try:
+        return _cache[type(pattern), pattern, flags]
+    except KeyError:
+        pass
     if isinstance(pattern, _pattern_type):
         if flags:
             raise ValueError(
@@ -271,12 +278,23 @@ def _compile(pattern, flags):
         return pattern
     if not sre_compile.isstring(pattern):
         raise TypeError("first argument must be string or compiled pattern")
-    return sre_compile.compile(pattern, flags)
+    p = sre_compile.compile(pattern, flags)
+    if len(_cache) >= _MAXCACHE:
+        _cache.clear()
+    _cache[type(pattern), pattern, flags] = p
+    return p
 
-@functools.lru_cache(maxsize=512)
 def _compile_repl(repl, pattern):
     # internal: compile replacement pattern
-    return sre_parse.parse_template(repl, pattern)
+    try:
+        return _cache_repl[repl, pattern]
+    except KeyError:
+        pass
+    p = sre_parse.parse_template(repl, pattern)
+    if len(_cache_repl) >= _MAXCACHE:
+        _cache_repl.clear()
+    _cache_repl[repl, pattern] = p
+    return p
 
 def _expand(pattern, match, template):
     # internal: match.expand implementation hook

Lib/test/test_imp.py

@@ -162,8 +162,10 @@ class ImportTests(unittest.TestCase):
 
             with warnings.catch_warnings():
                 warnings.simplefilter('ignore')
-                mod = imp.load_compiled(
-                    temp_mod_name, imp.cache_from_source(temp_mod_name + '.py'))
+                if not sys.dont_write_bytecode:
+                    mod = imp.load_compiled(
+                        temp_mod_name,
+                        imp.cache_from_source(temp_mod_name + '.py'))
             self.assertEqual(mod.a, 1)
 
             if not os.path.exists(test_package_name):

Lib/test/test_import.py

@@ -24,6 +24,10 @@ from test.support import (
 from test import script_helper
 
 
+skip_if_dont_write_bytecode = unittest.skipIf(
+        sys.dont_write_bytecode,
+        "test meaningful only when writing bytecode")
+
 def remove_files(name):
     for f in (name + ".py",
               name + ".pyc",
@@ -120,6 +124,7 @@ class ImportTests(unittest.TestCase):
         finally:
             del sys.path[0]
 
+    @skip_if_dont_write_bytecode
     def test_bug7732(self):
         source = TESTFN + '.py'
         os.mkdir(source)
@@ -230,6 +235,7 @@ class ImportTests(unittest.TestCase):
             remove_files(TESTFN)
             unload(TESTFN)
 
+    @skip_if_dont_write_bytecode
     def test_file_to_source(self):
         # check if __file__ points to the source file where available
         source = TESTFN + ".py"
@@ -316,6 +322,7 @@ class ImportTests(unittest.TestCase):
             self.fail("fromlist must allow bogus names")
 
 
+@skip_if_dont_write_bytecode
 class FilePermissionTests(unittest.TestCase):
     # tests for file mode on cached .pyc/.pyo files
 
@@ -642,6 +649,7 @@ class PycacheTests(unittest.TestCase):
         del sys.path[0]
         self._clean()
 
+    @skip_if_dont_write_bytecode
     def test_import_pyc_path(self):
         self.assertFalse(os.path.exists('__pycache__'))
         __import__(TESTFN)
@@ -654,6 +662,7 @@ class PycacheTests(unittest.TestCase):
                          "test meaningful only on posix systems")
     @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0,
             "due to varying filesystem permission semantics (issue #11956)")
+    @skip_if_dont_write_bytecode
     def test_unwritable_directory(self):
         # When the umask causes the new __pycache__ directory to be
         # unwritable, the import still succeeds but no .pyc file is written.
@@ -663,6 +672,7 @@ class PycacheTests(unittest.TestCase):
         self.assertFalse(os.path.exists(os.path.join(
             '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag))))
 
+    @skip_if_dont_write_bytecode
     def test_missing_source(self):
         # With PEP 3147 cache layout, removing the source but leaving the pyc
         # file does not satisfy the import.
@@ -673,6 +683,7 @@ class PycacheTests(unittest.TestCase):
         forget(TESTFN)
         self.assertRaises(ImportError, __import__, TESTFN)
 
+    @skip_if_dont_write_bytecode
     def test_missing_source_legacy(self):
         # Like test_missing_source() except that for backward compatibility,
         # when the pyc file lives where the py file would have been (and named
@@ -694,6 +705,7 @@ class PycacheTests(unittest.TestCase):
         pyc_file = imp.cache_from_source(TESTFN + '.py')
         self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file))
 
+    @skip_if_dont_write_bytecode
     def test___cached___legacy_pyc(self):
         # Like test___cached__() except that for backward compatibility,
         # when the pyc file lives where the py file would have been (and named
@@ -709,6 +721,7 @@ class PycacheTests(unittest.TestCase):
         self.assertEqual(m.__cached__,
                          os.path.join(os.curdir, os.path.relpath(pyc_file)))
 
+    @skip_if_dont_write_bytecode
     def test_package___cached__(self):
         # Like test___cached__ but for packages.
         def cleanup():

Lib/test/test_importlib/source/test_file_loader.py

@@ -159,7 +159,8 @@ class SimpleTest(unittest.TestCase):
         finally:
             os.unlink(file_path)
             pycache = os.path.dirname(imp.cache_from_source(file_path))
-            shutil.rmtree(pycache)
+            if os.path.exists(pycache):
+                shutil.rmtree(pycache)
 
     def test_timestamp_overflow(self):
         # When a modification timestamp is larger than 2**32, it should be

Lib/test/test_runpy.py

@@ -258,12 +258,13 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
             importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
-            make_legacy_pyc(mod_fname)
-            unload(mod_name)  # In case loader caches paths
-            importlib.invalidate_caches()
-            if verbose > 1: print("Running from compiled:", mod_name)
-            self._fix_ns_for_legacy_pyc(expected_ns, alter_sys)
-            self.check_code_execution(create_ns, expected_ns)
+            if not sys.dont_write_bytecode:
+                make_legacy_pyc(mod_fname)
+                unload(mod_name)  # In case loader caches paths
+                importlib.invalidate_caches()
+                if verbose > 1: print("Running from compiled:", mod_name)
+                self._fix_ns_for_legacy_pyc(expected_ns, alter_sys)
+                self.check_code_execution(create_ns, expected_ns)
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose > 1: print("Module executed successfully")
@@ -293,12 +294,13 @@ class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
             importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
-            make_legacy_pyc(mod_fname)
-            unload(mod_name)  # In case loader caches paths
-            if verbose > 1: print("Running from compiled:", pkg_name)
-            importlib.invalidate_caches()
-            self._fix_ns_for_legacy_pyc(expected_ns, alter_sys)
-            self.check_code_execution(create_ns, expected_ns)
+            if not sys.dont_write_bytecode:
+                make_legacy_pyc(mod_fname)
+                unload(mod_name)  # In case loader caches paths
+                if verbose > 1: print("Running from compiled:", pkg_name)
+                importlib.invalidate_caches()
+                self._fix_ns_for_legacy_pyc(expected_ns, alter_sys)
+                self.check_code_execution(create_ns, expected_ns)
         finally:
             self._del_pkg(pkg_dir, depth, pkg_name)
         if verbose > 1: print("Package executed successfully")
@@ -351,16 +353,17 @@ from ..uncle.cousin import nephew
             importlib.invalidate_caches()
             __import__(mod_name)
             os.remove(mod_fname)
-            make_legacy_pyc(mod_fname)
-            unload(mod_name)  # In case the loader caches paths
-            if verbose > 1: print("Running from compiled:", mod_name)
-            importlib.invalidate_caches()
-            d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
-            self.assertEqual(d2["__name__"], expected_name)
-            self.assertEqual(d2["__package__"], pkg_name)
-            self.assertIn("sibling", d2)
-            self.assertIn("nephew", d2)
-            del d2 # Ensure __loader__ entry doesn't keep file open
+            if not sys.dont_write_bytecode:
+                make_legacy_pyc(mod_fname)
+                unload(mod_name)  # In case the loader caches paths
+                if verbose > 1: print("Running from compiled:", mod_name)
+                importlib.invalidate_caches()
+                d2 = run_module(mod_name, run_name=run_name) # Read from bytecode
+                self.assertEqual(d2["__name__"], expected_name)
+                self.assertEqual(d2["__package__"], pkg_name)
+                self.assertIn("sibling", d2)
+                self.assertIn("nephew", d2)
+                del d2 # Ensure __loader__ entry doesn't keep file open
         finally:
             self._del_pkg(pkg_dir, depth, mod_name)
         if verbose > 1: print("Module executed successfully")
@@ -513,9 +516,10 @@ class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
             script_name = self._make_test_script(script_dir, mod_name)
             compiled_name = py_compile.compile(script_name, doraise=True)
             os.remove(script_name)
-            legacy_pyc = make_legacy_pyc(script_name)
-            self._check_script(script_dir, "<run_path>", legacy_pyc,
-                               script_dir)
+            if not sys.dont_write_bytecode:
+                legacy_pyc = make_legacy_pyc(script_name)
+                self._check_script(script_dir, "<run_path>", legacy_pyc,
+                                   script_dir)
 
     def test_directory_error(self):
         with temp_dir() as script_dir:

Misc/NEWS

@@ -283,6 +283,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #16389: Fixed a performance regression relative to Python 3.1 in the
+  caching of compiled regular expressions.
+
 - Added missing FeedParser and BytesFeedParser to email.parser.__all__.
 
 - Issue #17431: Fix missing import of BytesFeedParser in email.parser.
@@ -935,6 +938,9 @@ Extension Modules
 Tests
 -----
 
+- Issue #11420: make test suite pass with -B/DONTWRITEBYTECODE set.
+  Initial patch by Thomas Wouters.
+
 - Issue #10652: make tcl/tk tests run after __all__ test, patch by
   Zachary Ware.