Commit a6b993c8 authored by Kirill Smelkov's avatar Kirill Smelkov

gpython: Add way to run it with threads runtime

Until now gpython was always activating gevent on startup + adding
goodness such as "always UTF-8 default encoding"; go, chan, b/u and
friends available from builtin namespace, etc... While those goodness
are sometimes useful on their own, it is not always appropriate to force
a project to switch from threads to gevent.

For this reason add a flag to select which runtime - either gevent
or threads - gpython should use.

	gpython -Xgpython.runtime=gevent	selects gevent,

while

	gpython -Xgpython.runtime=threads	selects threads.

Gevent remains the default.

It is also possible to specify desired runtime via $GPYTHON_RUNTIME
environment variable.

/reviewed-on nexedi/pygolang!5
parent c0282565
......@@ -53,6 +53,8 @@ Additionally GPython sets UTF-8 to be default encoding always, and puts `go`,
GPython is optional and the rest of Pygolang can be used from under standard Python too.
However without gevent integration `go` spawns full - not lightweight - OS thread.
GPython can be also used with threads - not gevent - runtime. Please see
`GPython options`_ for details.
Goroutines and channels
......@@ -502,3 +504,22 @@ such benchmarking data in Python.
.. _golang.x.perf.benchlib: https://lab.nexedi.com/nexedi/pygolang/tree/master/golang/x/perf/benchlib.py
__ https://github.com/golang/proposal/blob/master/design/14313-benchmark-format.md
__ https://godoc.org/golang.org/x/perf/cmd/benchstat
--------
GPython options
---------------
GPython mimics and supports most of Python command-line options, like `gpython
-c <commands>` to run Python statements from command line, or `gpython -m
<module>` to execute a module. Such options have the same meaning as in
standard Python and are not documented here.
GPython-specific options and environment variables are listed below:
`-X gpython.runtime=(gevent|threads)`
Specify which runtime GPython should use. `gevent` provides lightweight
coroutines, while with `threads` `go` spawns full OS thread. `gevent` is
default. The runtime to use can be also specified via `$GPYTHON_RUNTIME`
environment variable.
......@@ -26,6 +26,9 @@ differences:
- gevent is pre-activated and stdlib is patched to be gevent aware;
- go, chan, select etc are put into builtin namespace;
- default string encoding is always set to UTF-8.
Gevent activation can be disabled via `-X gpython.runtime=threads`, or
$GPYTHON_RUNTIME=threads.
"""
# NOTE gpython is kept out of golang/ , since even just importing e.g. golang.cmd.gpython,
......@@ -75,8 +78,12 @@ def pymain(argv):
ver = []
if 'GPython' in sys.version:
golang = sys.modules['golang'] # must be already imported
gevent = sys.modules['gevent'] # must be already imported
gpyver = 'GPython %s [gevent %s]' % (golang.__version__, gevent.__version__)
gevent = sys.modules.get('gevent', None)
gpyver = 'GPython %s' % golang.__version__
if gevent is not None:
gpyver += ' [gevent %s]' % gevent.__version__
else:
gpyver += ' [threads]'
ver.append(gpyver)
import platform
......@@ -213,20 +220,65 @@ def main():
exe = exe + '.exe'
sys.executable = exe
# make gevent pre-available & stdlib patched
import gevent
from gevent import monkey
# XXX workaround for gevent vs pypy2 crash.
# XXX remove when gevent-1.4.1 is relased (https://github.com/gevent/gevent/pull/1357).
patch_thread=True
if pypy and sys.version_info.major == 2:
_ = monkey.patch_thread(existing_locks=False)
assert _ in (True, None)
patch_thread=False
_ = monkey.patch_all(thread=patch_thread) # XXX sys=True ?
if _ not in (True, None): # patched or nothing to do
# XXX provide details
raise RuntimeError('gevent monkey-patching failed')
# import os to get access to environment.
# it is practically ok to import os before gevent, because os is always
# imported by site. Yes, `import site` can be disabled by -S, but there is
# no harm wrt gevent monkey-patching even if we import os first.
import os
# extract and process -X
# -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
sys._xoptions = getattr(sys, '_xoptions', {})
argv_ = []
gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
while len(argv) > 0:
arg = argv[0]
argv = argv[1:]
if not arg.startswith('-X'):
argv_.append(arg)
# continue looking for -X only until options end
if not arg.startswith('-'):
break
continue
# -X <opt>
opt = arg[2:] # -X<opt>
if opt == '':
opt = argv[0] # -X <opt>
argv = argv[1:]
if opt.startswith('gpython.runtime='):
gpy_runtime = opt[len('gpython.runtime='):]
sys._xoptions['gpython.runtime'] = gpy_runtime
else:
raise RuntimeError('gpython: unknown -X option %s' % opt)
argv = argv_ + argv
# initialize according to selected runtime
if gpy_runtime == 'gevent':
# make gevent pre-available & stdlib patched
import gevent
from gevent import monkey
# XXX workaround for gevent vs pypy2 crash.
# XXX remove when gevent-1.4.1 is relased (https://github.com/gevent/gevent/pull/1357).
patch_thread=True
if pypy and sys.version_info.major == 2:
_ = monkey.patch_thread(existing_locks=False)
assert _ in (True, None)
patch_thread=False
_ = monkey.patch_all(thread=patch_thread) # XXX sys=True ?
if _ not in (True, None): # patched or nothing to do
# XXX provide details
raise RuntimeError('gevent monkey-patching failed')
gpy_verextra = 'gevent %s' % gevent.__version__
elif gpy_runtime == 'threads':
gpy_verextra = 'threads'
else:
raise RuntimeError('gpython: invalid runtime %s' % gpy_runtime)
# put go, chan, select, ... into builtin namespace
import golang
......@@ -235,7 +287,7 @@ def main():
setattr(builtins, k, getattr(golang, k))
# sys.version
sys.version += (' [GPython %s] [gevent %s]' % (golang.__version__, gevent.__version__))
sys.version += (' [GPython %s] [%s]' % (golang.__version__, gpy_verextra))
# tail to pymain
pymain(argv)
......
......@@ -33,6 +33,24 @@ is_cpython = (platform.python_implementation() == 'CPython')
# @gpython_only is marker to run a test only under gpython
gpython_only = pytest.mark.skipif('GPython' not in sys.version, reason="gpython-only test")
# runtime is pytest fixture that yields all variants of should be supported gpython runtimes:
# '' - not specified (gpython should autoselect)
# 'gevent'
# 'threads'
@pytest.fixture(scope="function", params=['', 'gevent', 'threads'])
def runtime(request):
yield request.param
# gpyenv returns environment appropriate for spawning gpython with
# specified runtime.
def gpyenv(runtime): # -> env
env = os.environ.copy()
if runtime != '':
env['GPYTHON_RUNTIME'] = runtime
else:
env.pop('GPYTHON_RUNTIME', None)
return env
@gpython_only
def test_defaultencoding_utf8():
......@@ -54,6 +72,12 @@ def test_golang_builtins():
@gpython_only
def test_gevent_activated():
# gpython, by default, acticates gevent.
# handling of various runtime modes is explicitly tested in test_Xruntime.
assert_gevent_activated()
def assert_gevent_activated():
assert 'gevent' in sys.modules
from gevent.monkey import is_module_patched as patched, is_object_patched as obj_patched
# builtin (gevent: only on py2 - on py3 __import__ uses fine-grained locking)
......@@ -91,15 +115,38 @@ def test_gevent_activated():
if sys.hexversion >= 0x03070000: # >= 3.7.0
assert patched('queue')
def assert_gevent_not_activated():
assert 'gevent' not in sys.modules
from gevent.monkey import is_module_patched as patched, is_object_patched as obj_patched
assert not patched('socket')
assert not patched('time')
assert not patched('select')
assert not patched('os')
assert not patched('signal')
assert not patched('thread' if PY2 else '_thread')
assert not patched('threading')
assert not patched('_threading_local')
assert not patched('ssl')
assert not patched('subprocess')
assert not patched('sys')
@gpython_only
def test_executable():
def test_executable(runtime):
# sys.executable must point to gpython and we must be able to execute it.
import gevent
assert 'gpython' in sys.executable
out = pyout(['-c', 'import sys; print(sys.version)'])
assert ('[GPython %s]' % golang.__version__) in str(out)
assert ('[gevent %s]' % gevent.__version__) in str(out)
ver = pyout(['-c', 'import sys; print(sys.version)'], env=gpyenv(runtime))
ver = str(ver)
assert ('[GPython %s]' % golang.__version__) in ver
if runtime != 'threads':
assert ('[gevent %s]' % gevent.__version__) in ver
assert ('[threads]') not in ver
else:
assert ('[gevent ') not in ver
assert ('[threads]') in ver
# verify pymain.
#
......@@ -138,11 +185,15 @@ def test_pymain():
# pymain -V/--version
# gpython_only because output differs from !gpython.
@gpython_only
def test_pymain_ver():
def test_pymain_ver(runtime):
from golang import b
from gpython import _version_info_str as V
import gevent
vok = 'GPython %s [gevent %s]' % (golang.__version__, gevent.__version__)
vok = 'GPython %s' % golang.__version__
if runtime != 'threads':
vok += ' [gevent %s]' % gevent.__version__
else:
vok += ' [threads]'
if is_cpython:
vok += ' / CPython %s' % platform.python_version()
......@@ -153,8 +204,29 @@ def test_pymain_ver():
vok += '\n'
ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE)
ret, out, err = _pyrun(['-V'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
assert (ret, out, b(err)) == (0, b'', b(vok))
ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE)
ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
assert (ret, out, b(err)) == (0, b'', b(vok))
# verify -X gpython.runtime=...
@gpython_only
def test_Xruntime(runtime):
env = os.environ.copy()
env.pop('GPYTHON_RUNTIME', None) # del
argv = []
if runtime != '':
argv += ['-X', 'gpython.runtime='+runtime]
prog = 'from gpython import gpython_test as t; '
if runtime != 'threads':
prog += 't.assert_gevent_activated(); '
else:
prog += 't.assert_gevent_not_activated(); '
prog += 'print("ok")'
argv += ['-c', prog]
out = pyout(argv, env=env)
assert out == b'ok\n'
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment