gpython: Implement -O

Let's teach gpython and pymain about -O because buildout calls `python -O`
to byte-compile python sources in optimized mode:

nexedi/slapos!862 (comment 121470)

When seeing -O, we take the approach to reexecute underlying python with
-O, so that e.g.

	gpython -O file.py

becomes executed as

	python -O gpython file.py

This ensures correctness. The same approach could be used to implement
support for e.g. -S and other low-level options.

An earlier attempt to implement -O without reexecution can be seen at

nexedi/pygolang!7

together with list of problems that arise via that way.

Original idea to reexecute itself come from @jerome.

gpython/__init__.py

@@ -41,7 +41,7 @@ $GPYTHON_RUNTIME=threads.
 from __future__ import print_function, absolute_import
 
 
-_pyopt = "c:m:VW:X:"
+_pyopt = "c:m:OVW:X:"
 _pyopt_long = ('version',)
 
 # pymain mimics `python ...`
@@ -91,9 +91,25 @@ def pymain(argv, init=None):
     run = None          # function to run according to -c/-m/file/interactive
     version = False     # set if `-V`
     warnoptions = []    # collected `-W arg`
+    reexec_with = []    # reexecute underlying python with those options (e.g. -O, -S, ...)
+    reexec_argv = []    # if reexecuting, reexecute with this application-level argv
 
     igetopt = _IGetOpt(argv, _pyopt, _pyopt_long)
     for (opt, arg) in igetopt:
+        # options that require reexecuting through underlying python with that -<opt>
+        if opt in (
+                '-O',   # optimize
+            ):
+            reexec_with.append(opt)
+            if arg is not None:
+                reexec_with.append(arg)
+            continue
+
+        reexec_argv.append(opt)
+        if arg is not None:
+            reexec_argv.append(arg)
+
+
         # -V / --version
         if opt in ('-V', '--version'):
             version = True
@@ -138,6 +154,7 @@ def pymain(argv, init=None):
             sys.exit(2)
 
     argv = igetopt.argv
+    reexec_argv += argv
     if run is None:
         # file
         if len(argv) > 0:
@@ -182,6 +199,20 @@ def pymain(argv, init=None):
 
 
     # ---- options processed -> start the interpreter ----
+
+    # reexec underlying interpreter on options that we cannot handle at python
+    # level after underlying interpreter is already started. For example
+    #
+    #   gpython -O file.py
+    #
+    # is reexecuted as
+    #
+    #   python -O gpython file.py
+    if len(reexec_with) > 0:
+        import os
+        argv = [sys._gpy_underlying_executable] + reexec_with + [sys.executable] + reexec_argv
+        os.execv(argv[0], argv)
+
     if init is not None:
         init()
 

gpython/gpython_test.py

@@ -230,15 +230,12 @@ def test_pymain_syspath():
     # check verifies that print_syspath output for gpython and underlying python is the same.
     # if path0cwd2realpath=Y, expect realpath('') instead of '' in sys.path[0]
     def check(argv, path0cwd2realpath=False, **kw):
-        gpyout   = u(pyout(argv, **kw))
-        stdpyout = u(pyout(argv, pyexe=sys._gpy_underlying_executable, **kw))
-        gpyoutv   = gpyout.splitlines()
-        stdpyoutv = stdpyout.splitlines()
-        if path0cwd2realpath:
-            assert stdpyoutv[0] == ''
-            stdpyoutv[0] = realpath(kw.get('cwd', ''))
+        def _(gpyoutv, stdpyoutv):
+            if path0cwd2realpath:
+                assert stdpyoutv[0] == ''
+                stdpyoutv[0] = realpath(kw.get('cwd', ''))
 
-        assert gpyoutv == stdpyoutv
+        check_gpy_vs_py(argv, postprocessf=_, **kw)
 
     check([], stdin=b'import print_syspath', cwd=testprog)  # interactive
     check(['-c', 'import print_syspath'], cwd=testprog)     # -c
@@ -246,6 +243,23 @@ def test_pymain_syspath():
             path0cwd2realpath=(PY2 or is_pypy))
     check(['testprog/print_syspath.py'], cwd=here)          # file
 
+
+# verify that pymain handles -O in exactly the same was as underlying python does.
+@gpython_only
+def test_pymain_opt():
+    def check(argv):
+        argv += ["print_opt.py"]
+        kw = {'cwd': testprog}
+        check_gpy_vs_py(argv, **kw)
+
+    check([])
+    check(["-O"])
+    check(["-OO"])
+    check(["-OOO"])
+    check(["-O", "-O"])
+    check(["-O", "-O", "-O"])
+
+
 # pymain -V/--version
 # gpython_only because output differs from !gpython.
 @gpython_only
@@ -318,3 +332,15 @@ def grepv(pattern, text): # -> text
         if not m:
             v.append(l)
     return t.join(v)
+
+# check_gpy_vs_py verifies that gpython output matches underlying python output.
+def check_gpy_vs_py(argv, postprocessf=None, **kw):
+        gpyout   = u(pyout(argv, **kw))
+        stdpyout = u(pyout(argv, pyexe=sys._gpy_underlying_executable, **kw))
+        gpyoutv   = gpyout.splitlines()
+        stdpyoutv = stdpyout.splitlines()
+
+        if postprocessf is not None:
+            postprocessf(gpyoutv, stdpyoutv)
+
+        assert gpyoutv == stdpyoutv

gpython/testprog/print_opt.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2020  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""Program print_opt prints information about optimizations."""
+
+from __future__ import print_function, absolute_import
+
+import sys, os, os.path, tempfile, shutil
+
+def main():
+    print('sys.flags.debug:      %s' % sys.flags.debug)
+    print('sys.flags.optimize:   %s' % sys.flags.optimize)
+    print('__debug__:            %s' % __debug__)
+    print('assert:               %s' % is_assert_enabled())
+    print('docstrings:           %s' % is_docstrings_enabled())
+    print('import mod.py:        %s' % modpy_imports_from())
+
+
+# is_assert_enabled returns whether assert statements are enabled.
+def is_assert_enabled():
+    try:
+        assert False # must raise AssertionError
+    except AssertionError:
+        return True
+    else:
+        return False
+
+# is_docstrings_enabled returns whether docstrings are enabled.
+def is_docstrings_enabled():
+    def _():
+        """hello world"""
+    if _.__doc__ is None:
+        return False
+    if _.__doc__ == "hello world":
+        return True
+    raise AssertionError(_.__doc__)
+
+# modpy returns name for compiled version of python module mod.py
+def modpy_imports_from():
+    try:
+        import mod
+    except ImportError:
+        # ok - should not be there
+        pass
+    else:
+        raise AssertionError("module 'mod' is already there")
+
+    tmpd = tempfile.mkdtemp('', 'modpy_imports_from')
+    try:
+        pymod = "%s/mod.py" % tmpd
+        with open(pymod, "w") as f:
+            f.write("# hello up there\n")
+
+        sys.path.insert(0, tmpd)
+        import mod
+
+        files = set()
+        for dirpath, dirnames, filenames in os.walk(tmpd):
+            for _ in filenames:
+                f = '%s/%s' % (dirpath, _)
+                if f.startswith(tmpd+'/'):
+                    f = f[len(tmpd+'/'):]
+                files.add(f)
+
+
+        files.remove("mod.py") # must be there | raises if not
+        if len(files) == 0:
+            from_ = "mod.py"   # source-only import
+        else:
+            if len(files) != 1:
+                raise AssertionError("mod.py -> multiple compiled files (%s)" % (files,))
+
+            from_ = files.pop()
+
+        return from_
+
+    finally:
+        shutil.rmtree(tmpd)
+
+
+if __name__ == '__main__':
+    main()