Merge pull request #1160 from gevent/issue1157

Use runpy.run_path for `python -m gevent.monkey`

CHANGES.rst

@@ -60,6 +60,10 @@ Enhancements
   time-sensitive parts of the hub itself. Please report any
   compatibility issues.
 
+- ``python -m gevent.monkey <script>`` accepts more values for
+  ``<script>``, including paths to packages or compiled bytecode.
+  Reported in :issue:`1157` by Eddie Linder.
+
 Monitoring and Debugging
 ------------------------
 

src/gevent/_util.py

@@ -86,12 +86,23 @@ def import_c_accel(globs, cname):
         # or we're running from the C extension
         return
 
-    import importlib
+
     from gevent._compat import PURE_PYTHON
     if PURE_PYTHON:
         return
 
-    mod = importlib.import_module(cname)
+    import importlib
+    import warnings
+    with warnings.catch_warnings():
+        # Python 3.7 likes to produce
+        # "ImportWarning: can't resolve
+        #   package from __spec__ or __package__, falling back on
+        #   __name__ and __path__"
+        # when we load cython compiled files. This is probably a bug in
+        # Cython, but it doesn't seem to have any consequences, it's
+        # just annoying to see and can mess up our unittests.
+        warnings.simplefilter('ignore', ImportWarning)
+        mod = importlib.import_module(cname)
 
     # By adopting the entire __dict__, we get a more accurate
     # __file__ and module repr, plus we don't leak any imported

src/gevent/local.py

@@ -153,8 +153,6 @@ affects what we see:
 """
 from __future__ import print_function
 
-import sys
-_PYPY = hasattr(sys, 'pypy_version_info')
 from copy import copy
 from weakref import ref
 
@@ -574,14 +572,20 @@ try:
     # in different ways. In CPython and PyPy3, it must be wrapped with classmethod;
     # in PyPy2, it must not. In either case, the args that get passed to
     # it are stil wrong.
-    if _PYPY and sys.version_info[:2] < (3, 0):
+    local.__new__ = 'None'
+except TypeError: # pragma: no cover
+    # Must be compiled
+    pass
+else:
+    from gevent._compat import PYPY
+    from gevent._compat import PY2
+    if PYPY and PY2:
         local.__new__ = __new__
     else:
         local.__new__ = classmethod(__new__)
-except TypeError: # pragma: no cover
-    pass
-finally:
-    del sys
+
+    del PYPY
+    del PY2
 
 _init()
 

src/gevent/monkey.py

@@ -766,13 +766,14 @@ def main():
     patch_all(**args)
     if argv:
         sys.argv = argv
-        __package__ = None
-        assert __package__ is None
-        globals()['__file__'] = sys.argv[0]  # issue #302
-        globals()['__package__'] = None # issue #975: make script be its own package
-        with open(sys.argv[0]) as f:
-            # Be sure to exec in globals to avoid import pollution. Also #975.
-            exec(f.read(), globals())
+        import runpy
+        # Use runpy.run_path to closely (exactly) match what the
+        # interpreter does given 'python <path>'. This includes allowing
+        # passing .pyc/.pyo files and packages with a __main__ and
+        # potentially even zip files. Previously we used exec, which only
+        # worked if we directly read a python source file.
+        runpy.run_path(sys.argv[0],
+                       run_name='__main__')
     else:
         print(script_help)
 
@@ -795,6 +796,12 @@ You can exclude a module with --no-module, e.g. --no-thread. You can
 specify a module to patch with --module, e.g. --socket. In the latter
 case only the modules specified on the command line will be patched.
 
+.. versionchanged:: 1.3b1
+    The *script* argument can now be any argument that can be passed to `runpy.run_path`,
+    just like the interpreter itself does, for example a package directory containing ``__main__.py``.
+    Previously it had to be the path to
+    a .py source file.
+
 MONKEY OPTIONS: --verbose %s""" % ', '.join('--[no-]%s' % m for m in modules)
     return script_help, patch_all_args, modules
 

src/greentest/monkey_package/__main__.py

+from __future__ import print_function
+# This file makes this directory into a package.
+# it exists to test 'python -m gevent.monkey monkey_package'
+print(__file__)
+print(__name__)

src/greentest/monkey_package/issue302monkey.py

+from __future__ import print_function
+import socket
+import sys
+if sys.argv[1] == 'patched':
+    print('gevent' in repr(socket.socket))
+else:
+    assert sys.argv[1] == 'stdlib'
+    print('gevent' not in repr(socket.socket))
+print(__file__)
+
+if sys.version_info[:2] == (2, 7):
+    # Prior to gevent 1.3, 'python -m gevent.monkey' guaranteed this to be
+    # None for all python versions.
+    print(__package__ == None)
+else:
+    if sys.argv[1] == 'patched':
+        # __package__ is handled differently, for some reason, and
+        # runpy doesn't let us override it. When we call it, it
+        # becomes ''. This appears to be against the documentation for
+        # runpy, which says specifically "If the supplied path
+        # directly references a script file (whether as source or as
+        # precompiled byte code), then __file__ will be set to the
+        # supplied path, and __spec__, __cached__, __loader__ and
+        # __package__ will all be set to None."
+        print(__package__ == '')
+    else:
+        # but the interpreter sets it to None
+        print(__package__ == None)

src/greentest/monkey_package/script.py

+# -*- coding: utf-8 -*-
+"""
+Test script file, to be used directly as a file.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+# We need some global imports
+from textwrap import dedent
+
+def use_import():
+    return dedent("    text")
+
+if __name__ == '__main__':
+    print(__file__)
+    print(__name__)
+    print(use_import())

src/greentest/test__issue302monkey.py

-import sys
-
-
-if 'gevent' not in sys.modules:
-    from subprocess import Popen
-    args = [sys.executable, '-m', 'gevent.monkey', __file__]
-    p = Popen(args)
-    code = p.wait()
-    assert code == 0, code
-
-else:
-    import socket
-    assert 'gevent' in repr(socket.socket), repr(socket.socket)
-    assert __file__ == 'test__issue302monkey.py', repr(__file__)
-    assert __package__ is None, __package__

src/greentest/test__monkey_scope.py

 import sys
 
+import unittest
 
-if 'gevent' not in sys.modules:
-    from subprocess import Popen
-    args = [sys.executable, '-m', 'gevent.monkey', __file__]
-    p = Popen(args)
-    code = p.wait()
-    assert code == 0, code
+from subprocess import Popen
+from subprocess import PIPE
 
-else:
-    from textwrap import dedent
+class TestRun(unittest.TestCase):
 
-    def use_import():
-        dedent("    text")
+    def _run(self, script):
+        args = [sys.executable, '-m', 'gevent.monkey', script, 'patched']
+        p = Popen(args, stdout=PIPE, stderr=PIPE)
+        gout, gerr = p.communicate()
+        self.assertEqual(0, p.returncode, (gout, gerr))
 
-    use_import()
+        args = [sys.executable, script, 'stdlib']
+        p = Popen(args, stdout=PIPE, stderr=PIPE)
+
+        pout, perr = p.communicate()
+        self.assertEqual(0, p.returncode, (pout, perr))
+
+        glines = gout.decode("utf-8").splitlines()
+        plines = pout.decode('utf-8').splitlines()
+        self.assertEqual(glines, plines)
+        self.assertEqual(gerr, perr)
+
+        return glines, gerr
+
+    def test_run_simple(self):
+        import os.path
+        self._run(os.path.join('monkey_package', 'script.py'))
+
+    def test_run_package(self):
+        # Run a __main__ inside a package.
+        lines, _ = self._run('monkey_package')
+
+        self.assertTrue(lines[0].endswith('__main__.py'), lines[0])
+        self.assertEqual(lines[1], '__main__')
+
+    def test_issue_302(self):
+        import os
+        lines, _ = self._run(os.path.join('monkey_package', 'issue302monkey.py'))
+
+        self.assertEqual(lines[0], 'True')
+        lines[1] = lines[1].replace('\\', '/') # windows path
+        self.assertEqual(lines[1], 'monkey_package/issue302monkey.py')
+        self.assertEqual(lines[2], 'True', lines)
+
+
+if __name__ == '__main__':
+    unittest.main()

src/greentest/tests_that_dont_do_leakchecks.txt

 test___monkey_patching.py
 test__monkey_ssl_warning.py
 test___monitor.py
+test__monkey_scope.py

src/greentest/tests_that_dont_monkeypatch.txt

@@ -18,3 +18,4 @@ test___ident.py
 test___config.py
 test___monitor.py
 test__events.py
+test__monkey_scope.py

src/greentest/tests_that_dont_use_resolver.txt

@@ -79,7 +79,6 @@ test__hub.py
 test__import_blocking_in_greenlet.py
 test__import_wait.py
 test__issue230.py
-test__issue302monkey.py
 test__issue330.py
 test__issue467.py
 test__issue6.py