Commit 8564dfdd authored by Kirill Smelkov's avatar Kirill Smelkov

gpython: Implement -O

Let's teach gpython and pymain about -O because buildout calls `python -O`
to byte-compile python sources in optimized mode:

nexedi/slapos!862 (comment 121470)

When seeing -O, we take the approach to reexecute underlying python with
-O, so that e.g.

	gpython -O file.py

becomes executed as

	python -O gpython file.py

This ensures correctness. The same approach could be used to implement
support for e.g. -S and other low-level options.

An earlier attempt to implement -O without reexecution can be seen at

nexedi/pygolang!7

together with list of problems that arise via that way.

Original idea to reexecute itself come from @jerome.
parent 076cdd8f
...@@ -41,7 +41,7 @@ $GPYTHON_RUNTIME=threads. ...@@ -41,7 +41,7 @@ $GPYTHON_RUNTIME=threads.
from __future__ import print_function, absolute_import from __future__ import print_function, absolute_import
_pyopt = "c:m:VW:X:" _pyopt = "c:m:OVW:X:"
_pyopt_long = ('version',) _pyopt_long = ('version',)
# pymain mimics `python ...` # pymain mimics `python ...`
...@@ -91,9 +91,25 @@ def pymain(argv, init=None): ...@@ -91,9 +91,25 @@ def pymain(argv, init=None):
run = None # function to run according to -c/-m/file/interactive run = None # function to run according to -c/-m/file/interactive
version = False # set if `-V` version = False # set if `-V`
warnoptions = [] # collected `-W arg` warnoptions = [] # collected `-W arg`
reexec_with = [] # reexecute underlying python with those options (e.g. -O, -S, ...)
reexec_argv = [] # if reexecuting, reexecute with this application-level argv
igetopt = _IGetOpt(argv, _pyopt, _pyopt_long) igetopt = _IGetOpt(argv, _pyopt, _pyopt_long)
for (opt, arg) in igetopt: for (opt, arg) in igetopt:
# options that require reexecuting through underlying python with that -<opt>
if opt in (
'-O', # optimize
):
reexec_with.append(opt)
if arg is not None:
reexec_with.append(arg)
continue
reexec_argv.append(opt)
if arg is not None:
reexec_argv.append(arg)
# -V / --version # -V / --version
if opt in ('-V', '--version'): if opt in ('-V', '--version'):
version = True version = True
...@@ -138,6 +154,7 @@ def pymain(argv, init=None): ...@@ -138,6 +154,7 @@ def pymain(argv, init=None):
sys.exit(2) sys.exit(2)
argv = igetopt.argv argv = igetopt.argv
reexec_argv += argv
if run is None: if run is None:
# file # file
if len(argv) > 0: if len(argv) > 0:
...@@ -182,6 +199,20 @@ def pymain(argv, init=None): ...@@ -182,6 +199,20 @@ def pymain(argv, init=None):
# ---- options processed -> start the interpreter ---- # ---- options processed -> start the interpreter ----
# reexec underlying interpreter on options that we cannot handle at python
# level after underlying interpreter is already started. For example
#
# gpython -O file.py
#
# is reexecuted as
#
# python -O gpython file.py
if len(reexec_with) > 0:
import os
argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
os.execv(argv[0], argv)
if init is not None: if init is not None:
init() init()
......
...@@ -230,15 +230,12 @@ def test_pymain_syspath(): ...@@ -230,15 +230,12 @@ def test_pymain_syspath():
# check verifies that print_syspath output for gpython and underlying python is the same. # check verifies that print_syspath output for gpython and underlying python is the same.
# if path0cwd2realpath=Y, expect realpath('') instead of '' in sys.path[0] # if path0cwd2realpath=Y, expect realpath('') instead of '' in sys.path[0]
def check(argv, path0cwd2realpath=False, **kw): def check(argv, path0cwd2realpath=False, **kw):
gpyout = u(pyout(argv, **kw)) def _(gpyoutv, stdpyoutv):
stdpyout = u(pyout(argv, pyexe=sys._gpy_underlying_executable, **kw)) if path0cwd2realpath:
gpyoutv = gpyout.splitlines() assert stdpyoutv[0] == ''
stdpyoutv = stdpyout.splitlines() stdpyoutv[0] = realpath(kw.get('cwd', ''))
if path0cwd2realpath:
assert stdpyoutv[0] == ''
stdpyoutv[0] = realpath(kw.get('cwd', ''))
assert gpyoutv == stdpyoutv check_gpy_vs_py(argv, postprocessf=_, **kw)
check([], stdin=b'import print_syspath', cwd=testprog) # interactive check([], stdin=b'import print_syspath', cwd=testprog) # interactive
check(['-c', 'import print_syspath'], cwd=testprog) # -c check(['-c', 'import print_syspath'], cwd=testprog) # -c
...@@ -246,6 +243,23 @@ def test_pymain_syspath(): ...@@ -246,6 +243,23 @@ def test_pymain_syspath():
path0cwd2realpath=(PY2 or is_pypy)) path0cwd2realpath=(PY2 or is_pypy))
check(['testprog/print_syspath.py'], cwd=here) # file check(['testprog/print_syspath.py'], cwd=here) # file
# verify that pymain handles -O in exactly the same was as underlying python does.
@gpython_only
def test_pymain_opt():
def check(argv):
argv += ["print_opt.py"]
kw = {'cwd': testprog}
check_gpy_vs_py(argv, **kw)
check([])
check(["-O"])
check(["-OO"])
check(["-OOO"])
check(["-O", "-O"])
check(["-O", "-O", "-O"])
# pymain -V/--version # pymain -V/--version
# gpython_only because output differs from !gpython. # gpython_only because output differs from !gpython.
@gpython_only @gpython_only
...@@ -318,3 +332,15 @@ def grepv(pattern, text): # -> text ...@@ -318,3 +332,15 @@ def grepv(pattern, text): # -> text
if not m: if not m:
v.append(l) v.append(l)
return t.join(v) return t.join(v)
# check_gpy_vs_py verifies that gpython output matches underlying python output.
def check_gpy_vs_py(argv, postprocessf=None, **kw):
gpyout = u(pyout(argv, **kw))
stdpyout = u(pyout(argv, pyexe=sys._gpy_underlying_executable, **kw))
gpyoutv = gpyout.splitlines()
stdpyoutv = stdpyout.splitlines()
if postprocessf is not None:
postprocessf(gpyoutv, stdpyoutv)
assert gpyoutv == stdpyoutv
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""Program print_opt prints information about optimizations."""
from __future__ import print_function, absolute_import
import sys, os, os.path, tempfile, shutil
def main():
print('sys.flags.debug: %s' % sys.flags.debug)
print('sys.flags.optimize: %s' % sys.flags.optimize)
print('__debug__: %s' % __debug__)
print('assert: %s' % is_assert_enabled())
print('docstrings: %s' % is_docstrings_enabled())
print('import mod.py: %s' % modpy_imports_from())
# is_assert_enabled returns whether assert statements are enabled.
def is_assert_enabled():
try:
assert False # must raise AssertionError
except AssertionError:
return True
else:
return False
# is_docstrings_enabled returns whether docstrings are enabled.
def is_docstrings_enabled():
def _():
"""hello world"""
if _.__doc__ is None:
return False
if _.__doc__ == "hello world":
return True
raise AssertionError(_.__doc__)
# modpy returns name for compiled version of python module mod.py
def modpy_imports_from():
try:
import mod
except ImportError:
# ok - should not be there
pass
else:
raise AssertionError("module 'mod' is already there")
tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
try:
pymod = "%s/mod.py" % tmpd
with open(pymod, "w") as f:
f.write("# hello up there\n")
sys.path.insert(0, tmpd)
import mod
files = set()
for dirpath, dirnames, filenames in os.walk(tmpd):
for _ in filenames:
f = '%s/%s' % (dirpath, _)
if f.startswith(tmpd+'/'):
f = f[len(tmpd+'/'):]
files.add(f)
files.remove("mod.py") # must be there | raises if not
if len(files) == 0:
from_ = "mod.py" # source-only import
else:
if len(files) != 1:
raise AssertionError("mod.py -> multiple compiled files (%s)" % (files,))
from_ = files.pop()
return from_
finally:
shutil.rmtree(tmpd)
if __name__ == '__main__':
main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment