Skip to content

pygolang
pygolang
Commit 9eb579f4 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

gpython/pymain: -O, sys.executable, ... 

- Teach gpython and pymain about `-`;
- Teach gpython and pymain about -O;
- Teach pymain to always set sys.executable, so that subprocess and multiprocessing work correctly out of the box;
- Fix ./bin/gpython crash;
- Rewrite options parsing via incremental getopt-like thing.

This changes are motivated by nexedi/slapos!862 (comment 121470) and nexedi/slapos!862 (comment 121600).

/reviewed-by @jerome
/reviewed-on nexedi/pygolang!10
parents 167912d3 840a5eae
Pipeline #12565 failed with stage
    in 0 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 376 additions and 116 deletions
    gpython/__init__.py
    View file @ 9eb579f4
    ......@@ -41,32 +41,84 @@ $GPYTHON_RUNTIME=threads.
    from __future__ import print_function, absolute_import
    _pyopt = "c:m:OVW:X:"
    _pyopt_long = ('version',)
    # pymain mimics `python ...`
    #
    # argv is what comes via `...` without first [0] for python.
    # argv is full argument vector including first [0] for path to main program itself.
    # init, if provided, is called after options are parsed, but before interpreter start.
    def pymain(argv, init=None):
    import sys
    from os.path import dirname, realpath
    # sys.executable
    # on windows there are
    # gpython-script.py
    # gpython.exe
    # gpython.manifest
    # and argv[0] is gpython-script.py
    exe = realpath(argv[0])
    argv = argv[1:]
    if exe.endswith('-script.py'):
    exe = exe[:-len('-script.py')]
    exe = exe + '.exe'
    sys._gpy_underlying_executable = sys.executable
    sys.executable = exe
    # `python /path/to/gpython` adds /path/to to sys.path[0] - remove it.
    # `gpython file` will add path-to-file to sys.path[0] by itself, and
    # /path/to/gpython is unneccessary and would create difference in behaviour
    # in between gpython and python.
    exedir = dirname(exe)
    if sys.path[0] == exedir:
    del sys.path[0]
    else:
    # buildout injects `sys.path[0:0] = eggs` into python scripts.
    # detect that and remove sys.path entry corresponding to exedir.
    if not _is_buildout_script(exe):
    raise RuntimeError('pymain: internal error: sys.path[0] was not set by underlying python to dirname(exe):'
    '\n\n\texe:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
    else:
    if exedir in sys.path:
    sys.path.remove(exedir)
    else:
    raise RuntimeError('pymain: internal error: sys.path does not contain dirname(exe):'
    '\n\n\texe:\t%s\n\tsys.path:\t%s' % (exe, sys.path))
    run = None # function to run according to -c/-m/file/interactive
    version = False # set if `-V`
    warnoptions = [] # collected `-W arg`
    reexec_with = [] # reexecute underlying python with those options (e.g. -O, -S, ...)
    reexec_argv = [] # if reexecuting, reexecute with this application-level argv
    igetopt = _IGetOpt(argv, _pyopt, _pyopt_long)
    for (opt, arg) in igetopt:
    # options that require reexecuting through underlying python with that -<opt>
    if opt in (
    '-O', # optimize
    ):
    reexec_with.append(opt)
    if arg is not None:
    reexec_with.append(arg)
    continue
    reexec_argv.append(opt)
    if arg is not None:
    reexec_argv.append(arg)
    while len(argv) > 0:
    # -V / --version
    if argv[0] in ('-V', '--version'):
    if opt in ('-V', '--version'):
    version = True
    break
    # -c command
    elif argv[0].startswith('-c'):
    cmd = argv[0][2:] # -c<command> also works
    argv = argv[1:]
    if cmd == '':
    cmd = argv[0]
    argv = argv[1:]
    sys.argv = ['-c'] + argv # python leaves '-c' as argv[0]
    elif opt == '-c':
    cmd = arg
    sys.argv = ['-c'] + igetopt.argv # python leaves '-c' as argv[0]
    sys.path.insert(0, '') # cwd
    def run():
    import six
    ......@@ -78,13 +130,9 @@ def pymain(argv, init=None):
    break
    # -m module
    elif argv[0].startswith('-m'):
    mod = argv[0][2:] # -m<module> also works
    argv = argv[1:]
    if mod == '':
    mod = argv[0]
    argv = argv[1:]
    sys.argv = [mod] + argv
    elif opt == '-m':
    mod = arg
    sys.argv = [mod] + igetopt.argv
    # sys.path <- cwd
    # NOTE python2 injects '', while python3 injects realpath('')
    # we stick to python3 behaviour, as it is more sane because e.g.
    ......@@ -98,20 +146,18 @@ def pymain(argv, init=None):
    break
    # -W arg (warning control)
    elif argv[0].startswith('-W'):
    wopt = argv[0][2:] # -W<arg> also works
    argv = argv[1:]
    if wopt == '':
    wopt = argv[0]
    argv = argv[1:]
    warnoptions.append(wopt)
    elif argv[0].startswith('-'):
    print("unknown option: '%s'" % argv[0], file=sys.stderr)
    elif opt == '-W':
    warnoptions.append(arg)
    else:
    print("unknown option: '%s'" % opt, file=sys.stderr)
    sys.exit(2)
    argv = igetopt.argv
    reexec_argv += argv
    if run is None:
    # file
    else:
    if len(argv) > 0 and argv[0] != '-':
    sys.argv = argv
    filepath = argv[0]
    ......@@ -124,36 +170,49 @@ def pymain(argv, init=None):
    '__doc__': None,
    '__package__': None}
    _execfile(filepath, g)
    break
    # interactive console
    if run is None:
    sys.argv = ['']
    sys.path.insert(0, '') # cwd
    def run():
    import code
    from six.moves import input as raw_input
    # like code.interact() but with overridden console.raw_input _and_
    # readline imported (code.interact mutually excludes those two).
    try:
    import readline # enable interactive editing
    except ImportError:
    pass
    console = code.InteractiveConsole()
    def _(prompt):
    # python behaviour: don't print '>>>' if stdin is not a tty
    # (builtin raw_input always prints prompt)
    if not sys.stdin.isatty():
    prompt=''
    return raw_input(prompt)
    console.raw_input = _
    console.interact()
    # interactive console
    else:
    sys.argv = [''] if len(argv) == 0 else argv # e.g. ['-']
    sys.path.insert(0, '') # cwd
    def run():
    import code
    from six.moves import input as raw_input
    # like code.interact() but with overridden console.raw_input _and_
    # readline imported (code.interact mutually excludes those two).
    try:
    import readline # enable interactive editing
    except ImportError:
    pass
    console = code.InteractiveConsole()
    def _(prompt):
    # python behaviour: don't print '>>>' if stdin is not a tty
    # (builtin raw_input always prints prompt)
    if not sys.stdin.isatty():
    prompt=''
    return raw_input(prompt)
    console.raw_input = _
    console.interact()
    # ---- options processed -> start the interpreter ----
    # reexec underlying interpreter on options that we cannot handle at python
    # level after underlying interpreter is already started. For example
    #
    # gpython -O file.py
    #
    # is reexecuted as
    #
    # python -O gpython file.py
    if len(reexec_with) > 0:
    import os
    argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
    os.execv(argv[0], argv)
    if init is not None:
    init()
    ......@@ -259,19 +318,6 @@ def main():
    sys.setdefaultencoding('utf-8')
    delattr(sys, 'setdefaultencoding')
    # sys.executable
    # on windows there are
    # gpython-script.py
    # gpython.exe
    # gpython.manifest
    # and argv[0] is gpython-script.py
    exe = sys.argv[0]
    argv = sys.argv[1:]
    if exe.endswith('-script.py'):
    exe = exe[:-len('-script.py')]
    exe = exe + '.exe'
    sys._gpy_underlying_executable = sys.executable
    sys.executable = exe
    # import os to get access to environment.
    # it is practically ok to import os before gevent, because os is always
    ......@@ -279,55 +325,33 @@ def main():
    # no harm wrt gevent monkey-patching even if we import os first.
    import os
    # `python /path/to/gpython` adds /path/to to sys.path[0] - remove it.
    # `gpython file` will add path-to-file to sys.path[0] by itself, and
    # /path/to/gpython is unneccessary and would create difference in behaviour
    # in between gpython and python.
    exedir = exe[:exe.rindex(os.sep)] # dirname, but os.path cannot be imported yet
    if sys.path[0] == exedir:
    del sys.path[0]
    else:
    # buildout injects `sys.path[0:0] = eggs` into python scripts.
    # detect that and remove sys.path entry corresponding to exedir.
    if not _is_buildout_script(exe):
    raise RuntimeError('gpython: internal error: sys.path[0] was not set by underlying python to dirname(gpython):'
    '\n\n\tgpython:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
    else:
    if exedir in sys.path:
    sys.path.remove(exedir)
    else:
    raise RuntimeError('gpython: internal error: sys.path does not contain dirname(gpython):'
    '\n\n\tgpython:\t%s\n\tsys.path:\t%s' % (exe, sys.path))
    # extract and process -X
    # extract and process `-X gpython.*`
    # -X gpython.runtime=(gevent|threads) + $GPYTHON_RUNTIME
    sys._xoptions = getattr(sys, '_xoptions', {})
    argv_ = []
    gpy_runtime = os.getenv('GPYTHON_RUNTIME', 'gevent')
    while len(argv) > 0:
    arg = argv[0]
    argv = argv[1:]
    igetopt = _IGetOpt(sys.argv[1:], _pyopt, _pyopt_long)
    for (opt, arg) in igetopt:
    if opt == '-X':
    if arg.startswith('gpython.'):
    if arg.startswith('gpython.runtime='):
    gpy_runtime = arg[len('gpython.runtime='):]
    sys._xoptions['gpython.runtime'] = gpy_runtime
    if not arg.startswith('-X'):
    argv_.append(arg)
    # continue looking for -X only until options end
    if not arg.startswith('-'):
    break
    continue
    else:
    raise RuntimeError('gpython: unknown -X option %s' % opt)
    # -X <opt>
    opt = arg[2:] # -X<opt>
    if opt == '':
    opt = argv[0] # -X <opt>
    argv = argv[1:]
    continue
    if opt.startswith('gpython.runtime='):
    gpy_runtime = opt[len('gpython.runtime='):]
    sys._xoptions['gpython.runtime'] = gpy_runtime
    argv_.append(opt)
    if arg is not None:
    argv_.append(arg)
    else:
    raise RuntimeError('gpython: unknown -X option %s' % opt)
    argv = argv_ + argv
    # options after -c / -m are not for python itself
    if opt in ('-c', '-m'):
    break
    argv = [sys.argv[0]] + argv_ + igetopt.argv
    # init initializes according to selected runtime
    # it is called after options are parsed and sys.path is setup correspondingly.
    ......@@ -382,5 +406,106 @@ def _is_buildout_script(path):
    return ('\nsys.path[0:0] = [\n' in src)
    # _IGetOpt provides getopt-style incremental options parsing.
    # ( we cannot use getopt directly, because it complains about "unrecognized options"
    # on e.g. `gpython file.py -opt` )
    class _IGetOpt:
    def __init__(self, argv, shortopts, longopts):
    self.argv = argv
    self._opts = {} # opt -> bool(arg-required)
    self._shortopttail = '' # current tail of short options from e.g. -abcd
    # parse shortopts -> ._opts
    opt = None
    for _ in shortopts:
    if _ == ':':
    if opt is None:
    raise RuntimeError("invalid shortopts: unexpected ':'")
    self._opts['-'+opt] = True
    opt = None # prevent ::
    else:
    opt = _
    if opt in self._opts:
    raise RuntimeError("invalid shortopts: double '%s'" % opt)
    self._opts['-'+opt] = False
    # parse longopts -> ._opts
    for opt in longopts:
    arg_required = (opt[-1:] == '=')
    if arg_required:
    opt = opt[:-1]
    self._opts['--'+opt] = arg_required
    def __iter__(self):
    return self
    def __next__(self):
    # yield e.g. -b -c -d from -abcd
    if len(self._shortopttail) > 0:
    opt = '-'+self._shortopttail[0]
    self._shortopttail = self._shortopttail[1:]
    if opt not in self._opts:
    raise RuntimeError('unexpected option %s' % opt)
    arg = None
    if self._opts[opt]: # arg required
    if len(self._shortopttail) > 0:
    # -o<arg>
    arg = self._shortopttail
    self._shortopttail = ''
    else:
    # -o <arg>
    if len(self.argv) == 0:
    raise RuntimeError('option %s requires an argument' % opt)
    arg = self.argv[0]
    self.argv = self.argv[1:]
    return (opt, arg)
    # ._shortopttail is empty - proceed with .argv
    if len(self.argv) == 0:
    raise StopIteration # end of argv
    opt = self.argv[0]
    if not opt.startswith('-'):
    raise StopIteration # not an option
    if opt == '-':
    raise StopIteration # not an option
    self.argv = self.argv[1:]
    if opt == '--':
    raise StopIteration # options -- args delimiter
    # short option
    if not opt.startswith('--'):
    self._shortopttail = opt[1:]
    return self.__next__()
    # long option
    arg = None
    if '=' in opt:
    opt, arg = opt.split('=')
    if opt not in self._opts:
    raise RuntimeError('unexpected option %s' % opt)
    arg_required = self._opts[opt]
    if not arg_required:
    if arg is not None:
    raise RuntimeError('option %s requires no argument' % opt)
    else:
    if arg is None:
    if len(self.argv) == 0:
    raise RuntimeError('option %s requires no argument' % opt)
    arg = self.argv[0]
    self.argv[0] = self.argv[1:]
    return (opt, arg)
    next = __next__ # for py2
    if __name__ == '__main__':
    main()
    gpython/gpython_test.py
    View file @ 9eb579f4
    ......@@ -163,6 +163,10 @@ def test_pymain():
    # interactive
    _ = pyout([], stdin=b'import hello\n', cwd=testdata)
    assert _ == b"hello\nworld\n['']\n"
    _ = pyout(['-'], stdin=b'import hello\n', cwd=testdata)
    assert _ == b"hello\nworld\n['-']\n"
    _ = pyout(['-', 'zzz'], stdin=b'import hello\n', cwd=testdata)
    assert _ == b"hello\nworld\n['-', 'zzz']\n"
    # -c <command>
    _ = pyout(['-c', 'import hello', 'abc', 'def'], cwd=testdata)
    ......@@ -230,15 +234,12 @@ def test_pymain_syspath():
    # check verifies that print_syspath output for gpython and underlying python is the same.
    # if path0cwd2realpath=Y, expect realpath('') instead of '' in sys.path[0]
    def check(argv, path0cwd2realpath=False, **kw):
    gpyout = u(pyout(argv, **kw))
    stdpyout = u(pyout(argv, pyexe=sys._gpy_underlying_executable, **kw))
    gpyoutv = gpyout.splitlines()
    stdpyoutv = stdpyout.splitlines()
    if path0cwd2realpath:
    assert stdpyoutv[0] == ''
    stdpyoutv[0] = realpath(kw.get('cwd', ''))
    def _(gpyoutv, stdpyoutv):
    if path0cwd2realpath:
    assert stdpyoutv[0] == ''
    stdpyoutv[0] = realpath(kw.get('cwd', ''))
    assert gpyoutv == stdpyoutv
    check_gpy_vs_py(argv, postprocessf=_, **kw)
    check([], stdin=b'import print_syspath', cwd=testprog) # interactive
    check(['-c', 'import print_syspath'], cwd=testprog) # -c
    ......@@ -246,6 +247,23 @@ def test_pymain_syspath():
    path0cwd2realpath=(PY2 or is_pypy))
    check(['testprog/print_syspath.py'], cwd=here) # file
    # verify that pymain handles -O in exactly the same was as underlying python does.
    @gpython_only
    def test_pymain_opt():
    def check(argv):
    argv += ["print_opt.py"]
    kw = {'cwd': testprog}
    check_gpy_vs_py(argv, **kw)
    check([])
    check(["-O"])
    check(["-OO"])
    check(["-OOO"])
    check(["-O", "-O"])
    check(["-O", "-O", "-O"])
    # pymain -V/--version
    # gpython_only because output differs from !gpython.
    @gpython_only
    ......@@ -274,6 +292,13 @@ def test_pymain_ver(runtime):
    ret, out, err = _pyrun(['--version'], stdout=PIPE, stderr=PIPE, env=gpyenv(runtime))
    assert (ret, out, b(err)) == (0, b'', b(vok))
    # verify that ./bin/gpython runs ok.
    @gpython_only
    def test_pymain_run_via_relpath():
    argv = ['-c', 'import sys; print(sys.version)']
    out1 = pyout( argv, pyexe=sys.executable)
    out2 = pyout(['./__init__.py'] + argv, pyexe=sys._gpy_underlying_executable, cwd=here)
    assert out1 == out2
    # verify -X gpython.runtime=...
    @gpython_only
    ......@@ -311,3 +336,15 @@ def grepv(pattern, text): # -> text
    if not m:
    v.append(l)
    return t.join(v)
    # check_gpy_vs_py verifies that gpython output matches underlying python output.
    def check_gpy_vs_py(argv, postprocessf=None, **kw):
    gpyout = u(pyout(argv, **kw))
    stdpyout = u(pyout(argv, pyexe=sys._gpy_underlying_executable, **kw))
    gpyoutv = gpyout.splitlines()
    stdpyoutv = stdpyout.splitlines()
    if postprocessf is not None:
    postprocessf(gpyoutv, stdpyoutv)
    assert gpyoutv == stdpyoutv
    gpython/testprog/print_opt.py 0 → 100644
    View file @ 9eb579f4
    # -*- coding: utf-8 -*-
    # Copyright (C) 2020 Nexedi SA and Contributors.
    # Kirill Smelkov <kirr@nexedi.com>
    #
    # This program is free software: you can Use, Study, Modify and Redistribute
    # it under the terms of the GNU General Public License version 3, or (at your
    # option) any later version, as published by the Free Software Foundation.
    #
    # You can also Link and Combine this program with other software covered by
    # the terms of any of the Free Software licenses or any of the Open Source
    # Initiative approved licenses and Convey the resulting work. Corresponding
    # source of such a combination shall include the source code for all other
    # software used.
    #
    # This program is distributed WITHOUT ANY WARRANTY; without even the implied
    # warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    #
    # See COPYING file for full licensing terms.
    # See https://www.nexedi.com/licensing for rationale and options.
    """Program print_opt prints information about optimizations."""
    from __future__ import print_function, absolute_import
    import sys, os, os.path, tempfile, shutil
    def main():
    print('sys.flags.debug: %s' % sys.flags.debug)
    print('sys.flags.optimize: %s' % sys.flags.optimize)
    print('__debug__: %s' % __debug__)
    print('assert: %s' % is_assert_enabled())
    print('docstrings: %s' % is_docstrings_enabled())
    print('import mod.py: %s' % modpy_imports_from())
    # is_assert_enabled returns whether assert statements are enabled.
    def is_assert_enabled():
    try:
    assert False # must raise AssertionError
    except AssertionError:
    return True
    else:
    return False
    # is_docstrings_enabled returns whether docstrings are enabled.
    def is_docstrings_enabled():
    def _():
    """hello world"""
    if _.__doc__ is None:
    return False
    if _.__doc__ == "hello world":
    return True
    raise AssertionError(_.__doc__)
    # modpy returns name for compiled version of python module mod.py
    def modpy_imports_from():
    try:
    import mod
    except ImportError:
    # ok - should not be there
    pass
    else:
    raise AssertionError("module 'mod' is already there")
    tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
    try:
    pymod = "%s/mod.py" % tmpd
    with open(pymod, "w") as f:
    f.write("# hello up there\n")
    sys.path.insert(0, tmpd)
    import mod
    files = set()
    for dirpath, dirnames, filenames in os.walk(tmpd):
    for _ in filenames:
    f = '%s/%s' % (dirpath, _)
    if f.startswith(tmpd+'/'):
    f = f[len(tmpd+'/'):]
    files.add(f)
    files.remove("mod.py") # must be there | raises if not
    if len(files) == 0:
    from_ = "mod.py" # source-only import
    else:
    if len(files) != 1:
    raise AssertionError("mod.py -> multiple compiled files (%s)" % (files,))
    from_ = files.pop()
    return from_
    finally:
    shutil.rmtree(tmpd)
    if __name__ == '__main__':
    main()
    Markdown is supported
    0% or
    You are about to add 0 people to the discussion. Proceed with caution.
    Finish editing this message first!
    Please register or to comment
    GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7