gpython: Fix pymain to properly setup sys.path (take 2)

This continues 6b4990f6 (gpython: Fix pymain to properly setup
sys.path[0]) - now we add test and verify that gpython actually sets
sys.path in the same way as underlying python does:

- don't add directory of gpython to sys.path
- for `gpython file` add to sys.path[0] realpath(dir(file)) instead of
  dir(file)  (see PySys_SetArgvEx for the reference, and else - if we do
  not make this change - added test breaks)
- for `gpython -m mod` add to sys.path[0] realpath('') instead of ''.
  This exactly matches PY3 behaviour, while PY2 behaviour is to add ''
  However PY3 behaviour

    * is more sane, and
    * fixes test_defer_excchain_traceback on py27-gevent

  -> so we stick to it.

Now all tests pass again.

golang/golang_test.py

@@ -1809,7 +1809,8 @@ def test_qq():
 # _pyrun runs `sys.executable argv... <stdin`.
 # it returns exit code, stdout and stderr.
 def _pyrun(argv, stdin=None, stdout=None, stderr=None, **kw):   # -> retcode, stdout, stderr
-    argv = [sys.executable] + argv
+    pyexe = kw.pop('pyexe', sys.executable)
+    argv  = [pyexe] + argv
 
     # adjust $PYTHONPATH to point to pygolang. This makes sure that external
     # script will succeed on `import golang` when running in-tree.

gpython/__init__.py

@@ -47,7 +47,7 @@ from __future__ import print_function, absolute_import
 # init, if provided, is called after options are parsed, but before interpreter start.
 def pymain(argv, init=None):
     import sys
-    from os.path import dirname
+    from os.path import dirname, realpath
 
     run = None          # function to run according to -c/-m/file/interactive
     version = False     # set if `-V`
@@ -85,7 +85,11 @@ def pymain(argv, init=None):
                 mod  = argv[0]
                 argv = argv[1:]
             sys.argv = [mod] + argv
-            sys.path.insert(0, '')  # cwd
+            # sys.path <- cwd
+            # NOTE python2 injects '', while python3 injects realpath('')
+            # we stick to python3 behaviour, as it is more sane because e.g.
+            # import path does not change after chdir.
+            sys.path.insert(0, realpath(''))  # realpath(cwd)
             def run():
                 import runpy
                 # search sys.path for module and run corresponding .py file as script
@@ -111,7 +115,7 @@ def pymain(argv, init=None):
             sys.argv = argv
             filepath = argv[0]
 
-            sys.path.insert(0, dirname(filepath))
+            sys.path.insert(0, realpath(dirname(filepath))) # not abspath -> see PySys_SetArgvEx
             def run():
                 # exec with same globals `python file.py` does
                 # XXX use runpy.run_path() instead?
@@ -266,6 +270,7 @@ def main():
     if exe.endswith('-script.py'):
         exe = exe[:-len('-script.py')]
         exe = exe + '.exe'
+    sys._gpy_underlying_executable = sys.executable
     sys.executable  = exe
 
     # import os to get access to environment.
@@ -274,6 +279,16 @@ def main():
     # no harm wrt gevent monkey-patching even if we import os first.
     import os
 
+    # `python /path/to/gpython` adds /path/to to sys.path[0] - remove it.
+    # `gpython file` will add path-to-file to sys.path[0] by itself, and
+    # /path/to/gpython is unneccessary and would create difference in behaviour
+    # in between gpython and python.
+    exedir = exe[:exe.rindex(os.sep)] # dirname, but os.path cannot be imported yet
+    if sys.path[0] != exedir:
+        raise RuntimeError('gpython: internal error: sys.path[0] was not set by underlying python to dirname(gpython):'
+                '\n\n\tgpython:\t%s\n\tsys.path[0]:\t%s' % (exe, sys.path[0]))
+    del sys.path[0]
+
     # extract and process -X
     # -X gpython.runtime=(gevent|threads)    + $GPYTHON_RUNTIME
     sys._xoptions = getattr(sys, '_xoptions', {})

gpython/gpython_test.py

@@ -27,6 +27,11 @@ from six import PY2
 from six.moves import builtins
 import pytest
 
+from os.path import join, dirname, realpath
+here     = dirname(__file__)
+testdata = join(here, 'testdata')
+testprog = join(here, 'testprog')
+
 is_pypy    = (platform.python_implementation() == 'PyPy')
 is_cpython = (platform.python_implementation() == 'CPython')
 
@@ -154,9 +159,6 @@ def test_executable(runtime):
 # gpython) and plain python (!gpython).
 def test_pymain():
     from golang import b
-    from os.path import join, dirname, realpath
-    here     = dirname(__file__)
-    testdata = join(dirname(__file__), 'testdata')
 
     # interactive
     _ = pyout([], stdin=b'import hello\n', cwd=testdata)
@@ -206,6 +208,28 @@ def test_pymain():
         b"- error::SyntaxWarning::*\n" + \
         b"- ignore::Warning::*\n"), _
 
+# verify that pymain sets sys.path in exactly the same way as underlying python does.
+@gpython_only
+def test_pymain_syspath():
+    # check verifies that print_syspath output for gpython and underlying python is the same.
+    # if path0cwd2realpath=Y, expect realpath('') instead of '' in sys.path[0]
+    def check(argv, path0cwd2realpath=False, **kw):
+        gpyout   = u(pyout(argv, **kw))
+        stdpyout = u(pyout(argv, pyexe=sys._gpy_underlying_executable, **kw))
+        gpyoutv   = gpyout.splitlines()
+        stdpyoutv = stdpyout.splitlines()
+        if path0cwd2realpath:
+            assert stdpyoutv[0] == ''
+            stdpyoutv[0] = realpath(kw.get('cwd', ''))
+
+        assert gpyoutv == stdpyoutv
+
+    check([], stdin=b'import print_syspath', cwd=testprog)  # interactive
+    check(['-c', 'import print_syspath'], cwd=testprog)     # -c
+    check(['-m', 'print_syspath'], cwd=testprog,            # -m
+            path0cwd2realpath=(PY2 or is_pypy))
+    check(['testprog/print_syspath.py'], cwd=here)          # file
+
 # pymain -V/--version
 # gpython_only because output differs from !gpython.
 @gpython_only

gpython/testprog/print_syspath.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2020  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""Program/module print_syspath prints value of sys.path"""
+
+from __future__ import print_function, absolute_import
+
+import sys
+
+def main():
+    for p in sys.path:
+        print(p)
+
+main()