Fix several issues relating to access to source code inside zipfiles. Initial...

Fix several issues relating to access to source code inside zipfiles. Initial work by Alexander Belopolsky. See Misc/NEWS in this checkin for details.

Lib/bdb.py

@@ -347,7 +347,7 @@ class Bdb:
             rv = frame.f_locals['__return__']
             s = s + '->'
             s = s + repr.repr(rv)
-        line = linecache.getline(filename, lineno)
+        line = linecache.getline(filename, lineno, frame.f_globals)
         if line: s = s + lprefix + line.strip()
         return s
 
@@ -589,7 +589,7 @@ class Tdb(Bdb):
         name = frame.f_code.co_name
         if not name: name = '???'
         fn = self.canonic(frame.f_code.co_filename)
-        line = linecache.getline(fn, frame.f_lineno)
+        line = linecache.getline(fn, frame.f_lineno, frame.f_globals)
         print '+++', fn, frame.f_lineno, name, ':', line.strip()
     def user_return(self, frame, retval):
         print '+++ return', retval

Lib/linecache.py

@@ -88,21 +88,20 @@ def updatecache(filename, module_globals=None):
             get_source = getattr(loader, 'get_source', None)
 
             if name and get_source:
-                if basename.startswith(name.split('.')[-1]+'.'):
-                    try:
-                        data = get_source(name)
-                    except (ImportError, IOError):
-                        pass
-                    else:
-                        if data is None:
-                            # No luck, the PEP302 loader cannot find the source
-                            # for this module.
-                            return []
-                        cache[filename] = (
-                            len(data), None,
-                            [line+'\n' for line in data.splitlines()], fullname
-                        )
-                        return cache[filename][2]
+                try:
+                    data = get_source(name)
+                except (ImportError, IOError):
+                    pass
+                else:
+                    if data is None:
+                        # No luck, the PEP302 loader cannot find the source
+                        # for this module.
+                        return []
+                    cache[filename] = (
+                        len(data), None,
+                        [line+'\n' for line in data.splitlines()], fullname
+                    )
+                    return cache[filename][2]
 
         # Try looking through the module search path.
 

Lib/pdb.py

@@ -440,7 +440,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         Return `lineno` if it is, 0 if not (e.g. a docstring, comment, blank
         line or EOF). Warning: testing is not comprehensive.
         """
-        line = linecache.getline(filename, lineno)
+        line = linecache.getline(filename, lineno, self.curframe.f_globals)
         if not line:
             print >>self.stdout, 'End of file'
             return 0
@@ -768,7 +768,7 @@ class Pdb(bdb.Bdb, cmd.Cmd):
         breaklist = self.get_file_breaks(filename)
         try:
             for lineno in range(first, last+1):
-                line = linecache.getline(filename, lineno)
+                line = linecache.getline(filename, lineno, self.curframe.f_globals)
                 if not line:
                     print >>self.stdout, '[EOF]'
                     break

Lib/runpy.py

@@ -65,13 +65,14 @@ def _run_module_code(code, init_globals=None,
 
 # This helper is needed due to a missing component in the PEP 302
 # loader protocol (specifically, "get_filename" is non-standard)
+# Since we can't introduce new features in maintenance releases,
+# support was added to zipimporter under the name '_get_filename'
 def _get_filename(loader, mod_name):
-    try:
-        get_filename = loader.get_filename
-    except AttributeError:
-        return None
-    else:
-        return get_filename(mod_name)
+    for attr in ("get_filename", "_get_filename"):
+        meth = getattr(loader, attr, None)
+        if meth is not None:
+            return meth(mod_name)
+    return None
 
 # Helper to get the loader, code and filename for a module
 def _get_module_details(mod_name):

Lib/test/test_cmd_line_script.py

@@ -75,36 +75,66 @@ def _compile_test_script(script_name):
         compiled_name = script_name + 'o'
     return compiled_name
 
-def _make_test_zip(zip_dir, zip_basename, script_name):
+def _make_test_zip(zip_dir, zip_basename, script_name, name_in_zip=None):
     zip_filename = zip_basename+os.extsep+'zip'
     zip_name = os.path.join(zip_dir, zip_filename)
     zip_file = zipfile.ZipFile(zip_name, 'w')
-    zip_file.write(script_name, os.path.basename(script_name))
+    if name_in_zip is None:
+        name_in_zip = os.path.basename(script_name)
+    zip_file.write(script_name, name_in_zip)
     zip_file.close()
-    # if verbose:
+    #if verbose:
     #    zip_file = zipfile.ZipFile(zip_name, 'r')
     #    print 'Contents of %r:' % zip_name
     #    zip_file.printdir()
     #    zip_file.close()
-    return zip_name
+    return zip_name, os.path.join(zip_name, name_in_zip)
 
 def _make_test_pkg(pkg_dir):
     os.mkdir(pkg_dir)
     _make_test_script(pkg_dir, '__init__', '')
 
+def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
+                       source=test_source, depth=1):
+    init_name = _make_test_script(zip_dir, '__init__', '')
+    init_basename = os.path.basename(init_name)
+    script_name = _make_test_script(zip_dir, script_basename, source)
+    pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)]
+    script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name))
+    zip_filename = zip_basename+os.extsep+'zip'
+    zip_name = os.path.join(zip_dir, zip_filename)
+    zip_file = zipfile.ZipFile(zip_name, 'w')
+    for name in pkg_names:
+        init_name_in_zip = os.path.join(name, init_basename)
+        zip_file.write(init_name, init_name_in_zip)
+    zip_file.write(script_name, script_name_in_zip)
+    zip_file.close()
+    os.unlink(init_name)
+    os.unlink(script_name)
+    #if verbose:
+    #    zip_file = zipfile.ZipFile(zip_name, 'r')
+    #    print 'Contents of %r:' % zip_name
+    #    zip_file.printdir()
+    #    zip_file.close()
+    return zip_name, os.path.join(zip_name, script_name_in_zip)
+
 # There's no easy way to pass the script directory in to get
 # -m to work (avoiding that is the whole point of making
 # directories and zipfiles executable!)
 # So we fake it for testing purposes with a custom launch script
 launch_source = """\
 import sys, os.path, runpy
-sys.path[0:0] = os.path.dirname(__file__)
+sys.path.insert(0, %s)
 runpy._run_module_as_main(%r)
 """
 
-def _make_launch_script(script_dir, script_basename, module_name):
-    return _make_test_script(script_dir, script_basename,
-                             launch_source % module_name)
+def _make_launch_script(script_dir, script_basename, module_name, path=None):
+    if path is None:
+        path = "os.path.dirname(__file__)"
+    else:
+        path = repr(path)
+    source = launch_source % (path, module_name)
+    return _make_test_script(script_dir, script_basename, source)
 
 class CmdLineTest(unittest.TestCase):
     def _check_script(self, script_name, expected_file,
@@ -155,15 +185,15 @@ class CmdLineTest(unittest.TestCase):
     def test_zipfile(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
-            zip_name = _make_test_zip(script_dir, 'test_zip', script_name)
-            self._check_script(zip_name, None, zip_name, '')
+            zip_name, run_name = _make_test_zip(script_dir, 'test_zip', script_name)
+            self._check_script(zip_name, run_name, zip_name, '')
 
     def test_zipfile_compiled(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             compiled_name = _compile_test_script(script_name)
-            zip_name = _make_test_zip(script_dir, 'test_zip', compiled_name)
-            self._check_script(zip_name, None, zip_name, '')
+            zip_name, run_name = _make_test_zip(script_dir, 'test_zip', compiled_name)
+            self._check_script(zip_name, run_name, zip_name, '')
 
     def test_module_in_package(self):
         with temp_dir() as script_dir:
@@ -171,8 +201,19 @@ class CmdLineTest(unittest.TestCase):
             _make_test_pkg(pkg_dir)
             script_name = _make_test_script(pkg_dir, 'script')
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script')
-            self._check_script(launch_name, script_name,
-                               script_name, 'test_pkg')
+            self._check_script(launch_name, script_name, script_name, 'test_pkg')
+
+    def test_module_in_package_in_zipfile(self):
+        with temp_dir() as script_dir:
+            zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script')
+            launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name)
+            self._check_script(launch_name, run_name, run_name, 'test_pkg')
+
+    def test_module_in_subpackage_in_zipfile(self):
+        with temp_dir() as script_dir:
+            zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2)
+            launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name)
+            self._check_script(launch_name, run_name, run_name, 'test_pkg.test_pkg')
 
 
 def test_main():

Lib/test/test_doctest.py

@@ -6,6 +6,9 @@ from test import test_support
 import doctest
 import warnings
 
+# NOTE: There are some additional tests relating to interaction with
+#       zipimport in the test_zipimport_support test module.
+
 ######################################################################
 ## Sample Objects (used by test cases)
 ######################################################################
@@ -369,7 +372,7 @@ We'll simulate a __file__ attr that ends in pyc:
     >>> tests = finder.find(sample_func)
 
     >>> print tests  # doctest: +ELLIPSIS
-    [<DocTest sample_func from ...:13 (1 example)>]
+    [<DocTest sample_func from ...:16 (1 example)>]
 
 The exact name depends on how test_doctest was invoked, so allow for
 leading path components.

Lib/test/test_inspect.py

@@ -16,6 +16,9 @@ from test import inspect_fodder2 as mod2
 # getclasstree, getargspec, getargvalues, formatargspec, formatargvalues,
 # currentframe, stack, trace, isdatadescriptor
 
+# NOTE: There are some additional tests relating to interaction with
+#       zipimport in the test_zipimport_support test module.
+
 modfile = mod.__file__
 if modfile.endswith(('c', 'o')):
     modfile = modfile[:-1]

Lib/test/test_zipimport.py

@@ -214,16 +214,24 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             zi = zipimport.zipimporter(TEMP_ZIP)
             self.assertEquals(zi.archive, TEMP_ZIP)
             self.assertEquals(zi.is_package(TESTPACK), True)
-            zi.load_module(TESTPACK)
+            mod = zi.load_module(TESTPACK)
+            self.assertEquals(zi._get_filename(TESTPACK), mod.__file__)
 
             self.assertEquals(zi.is_package(packdir + '__init__'), False)
             self.assertEquals(zi.is_package(packdir + TESTPACK2), True)
             self.assertEquals(zi.is_package(packdir2 + TESTMOD), False)
 
-            mod_name = packdir2 + TESTMOD
-            mod = __import__(module_path_to_dotted_name(mod_name))
+            mod_path = packdir2 + TESTMOD
+            mod_name = module_path_to_dotted_name(mod_path)
+            pkg = __import__(mod_name)
+            mod = sys.modules[mod_name]
             self.assertEquals(zi.get_source(TESTPACK), None)
-            self.assertEquals(zi.get_source(mod_name), None)
+            self.assertEquals(zi.get_source(mod_path), None)
+            self.assertEquals(zi._get_filename(mod_path), mod.__file__)
+            # To pass in the module name instead of the path, we must use the right importer
+            loader = mod.__loader__
+            self.assertEquals(loader.get_source(mod_name), None)
+            self.assertEquals(loader._get_filename(mod_name), mod.__file__)
 
             # test prefix and archivepath members
             zi2 = zipimport.zipimporter(TEMP_ZIP + os.sep + TESTPACK)
@@ -251,15 +259,23 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
             self.assertEquals(zi.archive, TEMP_ZIP)
             self.assertEquals(zi.prefix, packdir)
             self.assertEquals(zi.is_package(TESTPACK2), True)
-            zi.load_module(TESTPACK2)
+            mod = zi.load_module(TESTPACK2)
+            self.assertEquals(zi._get_filename(TESTPACK2), mod.__file__)
 
             self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False)
             self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False)
 
-            mod_name = TESTPACK2 + os.sep + TESTMOD
-            mod = __import__(module_path_to_dotted_name(mod_name))
+            mod_path = TESTPACK2 + os.sep + TESTMOD
+            mod_name = module_path_to_dotted_name(mod_path)
+            pkg = __import__(mod_name)
+            mod = sys.modules[mod_name]
             self.assertEquals(zi.get_source(TESTPACK2), None)
-            self.assertEquals(zi.get_source(mod_name), None)
+            self.assertEquals(zi.get_source(mod_path), None)
+            self.assertEquals(zi._get_filename(mod_path), mod.__file__)
+            # To pass in the module name instead of the path, we must use the right importer
+            loader = mod.__loader__
+            self.assertEquals(loader.get_source(mod_name), None)
+            self.assertEquals(loader._get_filename(mod_name), mod.__file__)
         finally:
             z.close()
             os.remove(TEMP_ZIP)

Misc/NEWS

@@ -74,6 +74,25 @@ Core and Builtins
 Library
 -------
 
+- Issue #4223: inspect.getsource() will now correctly display source code
+  for packages loaded via zipimport (or any other conformant PEP 302
+  loader). Original patch by Alexander Belopolsky.
+
+- Issue #4201: pdb can now access and display source code loaded via
+  zipimport (or any other conformant PEP 302 loader). Original patch by
+  Alexander Belopolsky.
+
+- Issue #4197: doctests in modules loaded via zipimport (or any other PEP
+  302 conformant loader) will now work correctly in most cases (they
+  are still subject to the constraints that exist for all code running
+  from inside a module loaded via a PEP 302 loader and attempting to
+  perform IO operations based on __file__). Original patch by
+  Alexander Belopolsky.
+
+- Issues #4082 and #4512: Add runpy support to zipimport in a manner that
+  allows backporting to maintenance branches. Original patch by
+  Alexander Belopolsky.
+
 - Issue #4163: Use unicode-friendly word splitting in the textwrap functions
   when given an unicode string.
 

Modules/zipimport.c

@@ -369,6 +369,29 @@ error:
 	return NULL;
 }
 
+/* Return a string matching __file__ for the named module */
+static PyObject *
+zipimporter_get_filename(PyObject *obj, PyObject *args)
+{
+    ZipImporter *self = (ZipImporter *)obj;
+    PyObject *code;
+    char *fullname, *modpath;
+    int ispackage;
+
+    if (!PyArg_ParseTuple(args, "s:zipimporter._get_filename",
+                         &fullname))
+        return NULL;
+
+    /* Deciding the filename requires working out where the code
+       would come from if the module was actually loaded */
+    code = get_module_code(self, fullname, &ispackage, &modpath);
+    if (code == NULL)
+        return NULL;
+    Py_DECREF(code); /* Only need the path info */
+
+    return PyString_FromString(modpath);
+}
+
 /* Return a bool signifying whether the module is a package or not. */
 static PyObject *
 zipimporter_is_package(PyObject *obj, PyObject *args)
@@ -528,6 +551,12 @@ Return the source code for the specified module. Raise ZipImportError\n\
 is the module couldn't be found, return None if the archive does\n\
 contain the module, but has no source for it.");
 
+
+PyDoc_STRVAR(doc_get_filename,
+"_get_filename(fullname) -> filename string.\n\
+\n\
+Return the filename for the specified module.");
+
 static PyMethodDef zipimporter_methods[] = {
 	{"find_module", zipimporter_find_module, METH_VARARGS,
 	 doc_find_module},
@@ -539,6 +568,8 @@ static PyMethodDef zipimporter_methods[] = {
 	 doc_get_code},
 	{"get_source", zipimporter_get_source, METH_VARARGS,
 	 doc_get_source},
+	{"_get_filename", zipimporter_get_filename, METH_VARARGS,
+	 doc_get_filename},
 	{"is_package", zipimporter_is_package, METH_VARARGS,
 	 doc_is_package},
 	{NULL,		NULL}	/* sentinel */