Issue #19081: Remove the zipimporter.files reference as the zip TOC

caches are module global in the zip_directory_cache. When it is
updated due to a changed zip file, all zipimporter instances need to
see the same updates TOC cache.  This fixes the bug for the overlooked
submodule import case from the earlier round of changes.  Includes
tests that would fail otherwise.

It also refactors zipimporter_init in the process to make it a bit
easier to read and understand.  Less reuse of the same variable for
multiple purposes and the local path buffer is malloc'ed instead
of consuming a large MAXPATHLEN+2 chunk stack space.

Lib/test/test_zipimport.py

@@ -395,59 +395,144 @@ class ZipFileModifiedAfterImportTestCase(ImportHooksBaseTestCase):
     def setUp(self):
         zipimport._zip_directory_cache.clear()
         zipimport._zip_stat_cache.clear()
+        # save sys.modules so we can unimport everything done by our tests.
+        self._sys_modules_orig = dict(sys.modules)
         ImportHooksBaseTestCase.setUp(self)
 
     def tearDown(self):
         ImportHooksBaseTestCase.tearDown(self)
+        # The closest we can come to un-importing our zipped up test modules.
+        sys.modules.clear()
+        sys.modules.update(self._sys_modules_orig)
         if os.path.exists(TEMP_ZIP):
             os.remove(TEMP_ZIP)
 
-    def testZipFileChangesAfterFirstImport(self):
-        """Alter the zip file after caching its index and try an import."""
+    def setUpZipFileModuleAndTestImports(self):
+        # Create a .zip file to test with
+        self.zipfile_path = TEMP_ZIP
         packdir = TESTPACK + os.sep
         files = {packdir + "__init__" + pyc_ext: (NOW, test_pyc),
                  packdir + TESTMOD + ".py": (NOW, "test_value = 38\n"),
                  "ziptest_a.py": (NOW, "test_value = 23\n"),
                  "ziptest_b.py": (NOW, "test_value = 42\n"),
                  "ziptest_c.py": (NOW, "test_value = 1337\n")}
-        zipfile_path = TEMP_ZIP
-        _write_zip_package(zipfile_path, files)
-        self.assertTrue(os.path.exists(zipfile_path))
-        sys.path.insert(0, zipfile_path)
+        _write_zip_package(self.zipfile_path, files)
+        self.assertTrue(os.path.exists(self.zipfile_path))
+        sys.path.insert(0, self.zipfile_path)
+
+        self.testpack_testmod = TESTPACK + "." + TESTMOD
+
+        with io.open(self.zipfile_path, "rb") as orig_zip_file:
+            self.orig_zip_file_contents = orig_zip_file.read()
 
         # Import something out of the zipfile and confirm it is correct.
-        testmod = __import__(TESTPACK + "." + TESTMOD,
+        testmod = __import__(self.testpack_testmod,
                              globals(), locals(), ["__dummy__"])
         self.assertEqual(testmod.test_value, 38)
+        del sys.modules[TESTPACK]
+        del sys.modules[self.testpack_testmod]
+
         # Import something else out of the zipfile and confirm it is correct.
         ziptest_b = __import__("ziptest_b", globals(), locals(), ["test_value"])
         self.assertEqual(ziptest_b.test_value, 42)
+        del sys.modules["ziptest_b"]
 
-        # Truncate and fill the zip file with non-zip garbage.
-        with io.open(zipfile_path, "rb") as orig_zip_file:
-            orig_zip_file_contents = orig_zip_file.read()
-        with io.open(zipfile_path, "wb") as byebye_valid_zip_file:
+    def truncateAndFillZipWithNonZipGarbage(self):
+        with io.open(self.zipfile_path, "wb") as byebye_valid_zip_file:
             byebye_valid_zip_file.write(b"Tear down this wall!\n"*1987)
+
+    def restoreZipFileWithDifferentHeaderOffsets(self):
+        """Make it a valid zipfile with some garbage at the start."""
+        # This alters all of the caches offsets within the file.
+        with io.open(self.zipfile_path, "wb") as new_zip_file:
+            new_zip_file.write(b"X"*1991)  # The year Python was created.
+            new_zip_file.write(self.orig_zip_file_contents)
+
+    def testZipFileChangesAfterFirstImport(self):
+        """Alter the zip file after caching its index and try an import."""
+        self.setUpZipFileModuleAndTestImports()
+        # The above call cached the .zip table of contents during its tests.
+        self.truncateAndFillZipWithNonZipGarbage()
         # Now that the zipfile has been replaced, import something else from it
         # which should fail as the file contents are now garbage.
         with self.assertRaises(ImportError):
             ziptest_a = __import__("ziptest_a", globals(), locals(),
                                    ["test_value"])
+        # The code path used by the __import__ call is different than
+        # that used by import statements.  Try these as well.  Some of
+        # these may create new zipimporter instances.  We need to
+        # function properly using the global zipimport caches
+        # regardless of how many zipimporter instances for the same
+        # .zip file exist.
+        with self.assertRaises(ImportError):
+            import ziptest_a
+        with self.assertRaises(ImportError):
+            from ziptest_a import test_value
+        with self.assertRaises(ImportError):
+            exec("from {} import {}".format(TESTPACK, TESTMOD))
 
-        # Now lets make it a valid zipfile that has some garbage at the start.
-        # This alters all of the offsets within the file
-        with io.open(zipfile_path, "wb") as new_zip_file:
-            new_zip_file.write(b"X"*1991)  # The year Python was created.
-            new_zip_file.write(orig_zip_file_contents)
+        # Alters all of the offsets within the file
+        self.restoreZipFileWithDifferentHeaderOffsets()
 
         # Now that the zip file has been "restored" to a valid but different
-        # zipfile the zipimporter should *successfully* re-read the new zip
-        # file's end of file central index and be able to import from it again.
+        # zipfile all zipimporter instances should *successfully* re-read the
+        # new file's end of file central index and be able to import again.
+
+        # Importing a submodule triggers a different import code path.
+        exec("import " + self.testpack_testmod)
+        self.assertEqual(getattr(locals()[TESTPACK], TESTMOD).test_value, 38)
+        exec("from {} import {}".format(TESTPACK, TESTMOD))
+        self.assertEqual(locals()[TESTMOD].test_value, 38)
+
         ziptest_a = __import__("ziptest_a", globals(), locals(), ["test_value"])
         self.assertEqual(ziptest_a.test_value, 23)
         ziptest_c = __import__("ziptest_c", globals(), locals(), ["test_value"])
         self.assertEqual(ziptest_c.test_value, 1337)
 
+    def testZipFileSubpackageImport(self):
+        """Import via multiple sys.path entries into parts of the zip."""
+        self.setUpZipFileModuleAndTestImports()
+        # Put a subdirectory within the zip file into the import path.
+        sys.path.insert(0, self.zipfile_path + os.sep + TESTPACK)
+
+        testmod = __import__(TESTMOD, globals(), locals(), ["test_value"])
+        self.assertEqual(testmod.test_value, 38)
+        del sys.modules[TESTMOD]
+        exec("from {} import test_value".format(TESTMOD))
+        self.assertEqual(test_value, 38)
+        del sys.modules[TESTMOD]
+
+        # Confirm that imports from the top level of the zip file
+        # (already in sys.path from the setup call above) still work.
+        ziptest_a = __import__("ziptest_a", globals(), locals(), ["test_value"])
+        self.assertEqual(ziptest_a.test_value, 23)
+        del sys.modules["ziptest_a"]
+        import ziptest_c
+        self.assertEqual(ziptest_c.test_value, 1337)
+        del sys.modules["ziptest_c"]
+
+        self.truncateAndFillZipWithNonZipGarbage()
+        # Imports should now fail.
+        with self.assertRaises(ImportError):
+            testmod = __import__(TESTMOD, globals(), locals(), ["test_value"])
+        with self.assertRaises(ImportError):
+            exec("from {} import test_value".format(TESTMOD))
+        with self.assertRaises(ImportError):
+            import ziptest_a
+
+        self.restoreZipFileWithDifferentHeaderOffsets()
+        # Imports should work again, the central directory TOC will be re-read.
+        testmod = __import__(TESTMOD, globals(), locals(), ["test_value"])
+        self.assertEqual(testmod.test_value, 38)
+        del sys.modules[TESTMOD]
+        exec("from {} import test_value".format(TESTMOD))
+        self.assertEqual(test_value, 38)
+
+        ziptest_a = __import__("ziptest_a", globals(), locals(), ["test_value"])
+        self.assertEqual(ziptest_a.test_value, 23)
+        import ziptest_c
+        self.assertEqual(ziptest_c.test_value, 1337)
+
 
 class BadFileZipImportTestCase(unittest.TestCase):
     def assertZipFailure(self, filename):