bpo-36763: Add _Py_InitializeMain() (GH-13362)

* Add a private _Py_InitializeMain() function.
* Add again _PyCoreConfig._init_main.
* _Py_InitializeFromConfig() now uses _init_main to decide
  if _Py_InitializeMainInterpreter() should be called.
* _PyCoreConfig: rename _frozen to pathconfig_warnings, its value is
  now the opposite of Py_FrozenFlag.
* Add an unit test for _init_main=0 and _Py_InitializeMain().

Include/cpython/coreconfig.h

@@ -398,10 +398,14 @@ typedef struct {
        See PEP 552 "Deterministic pycs" for more details. */
     wchar_t *check_hash_pycs_mode;
 
-    /* If greater than 0, suppress _PyPathConfig_Calculate() warnings.
+    /* If greater than 0, suppress _PyPathConfig_Calculate() warnings on Unix.
+       The parameter has no effect on Windows.
 
-       If set to -1 (default), inherit Py_FrozenFlag value. */
-    int _frozen;
+       If set to -1 (default), inherit !Py_FrozenFlag value. */
+    int pathconfig_warnings;
+
+    /* If equal to 0, stop Python initialization before the "main" phase */
+    int _init_main;
 
 } _PyCoreConfig;
 
@@ -438,7 +442,8 @@ typedef struct {
         .buffered_stdio = -1, \
         ._install_importlib = 1, \
         .check_hash_pycs_mode = NULL, \
-        ._frozen = -1}
+        .pathconfig_warnings = -1, \
+        ._init_main = 1}
 /* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */
 
 #ifdef __cplusplus

Include/cpython/pylifecycle.h

@@ -40,6 +40,7 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeFromWideArgs(
     const _PyCoreConfig *config,
     int argc,
     wchar_t **argv);
+PyAPI_FUNC(_PyInitError) _Py_InitializeMain(void);
 
 PyAPI_FUNC(int) _Py_RunMain(void);
 

Lib/test/test_embed.py

@@ -343,7 +343,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         '_install_importlib': 1,
         'check_hash_pycs_mode': 'default',
-        '_frozen': 0,
+        'pathconfig_warnings': 1,
+        '_init_main': 1,
     }
     if MS_WINDOWS:
         DEFAULT_PRE_CONFIG.update({
@@ -371,7 +372,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         ('Py_DontWriteBytecodeFlag', 'write_bytecode', True),
         ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'),
         ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'),
-        ('Py_FrozenFlag', '_frozen'),
+        ('Py_FrozenFlag', 'pathconfig_warnings', True),
         ('Py_IgnoreEnvironmentFlag', 'use_environment', True),
         ('Py_InspectFlag', 'inspect'),
         ('Py_InteractiveFlag', 'interactive'),
@@ -500,7 +501,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         self.assertEqual(config['global_config'], expected)
 
-    def check_config(self, testname, expected_config, expected_preconfig, add_path=None):
+    def check_config(self, testname, expected_config, expected_preconfig,
+                     add_path=None, stderr=None):
         env = dict(os.environ)
         # Remove PYTHON* environment variables to get deterministic environment
         for key in list(env):
@@ -511,19 +513,22 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         env['PYTHONCOERCECLOCALE'] = '0'
         env['PYTHONUTF8'] = '0'
 
-        out, err = self.run_embedded_interpreter(testname, env=env)
-        # Ignore err
-        try:
-            config = json.loads(out)
-        except json.JSONDecodeError:
-            self.fail(f"fail to decode stdout: {out!r}")
-
         expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig)
         expected_config = self.get_expected_config(expected_config, env, add_path)
         for key in self.COPY_PRE_CONFIG:
             if key not in expected_preconfig:
                 expected_preconfig[key] = expected_config[key]
 
+        out, err = self.run_embedded_interpreter(testname, env=env)
+        if stderr is None and not expected_config['verbose']:
+            stderr = ""
+        if stderr is not None:
+            self.assertEqual(err.rstrip(), stderr)
+        try:
+            config = json.loads(out)
+        except json.JSONDecodeError:
+            self.fail(f"fail to decode stdout: {out!r}")
+
         self.check_pre_config(config, expected_preconfig)
         self.check_core_config(config, expected_config)
         self.check_global_config(config)
@@ -689,7 +694,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         self.check_config("init_read_set", core_config, preconfig,
                           add_path="init_read_set_path")
 
-    def test_run_main_config(self):
+    def test_init_run_main(self):
+        preconfig = {}
+        code = ('import _testinternalcapi, json; '
+                'print(json.dumps(_testinternalcapi.get_configs()))')
+        core_config = {
+            'argv': ['-c', 'arg2'],
+            'program': 'python3',
+            'program_name': './python3',
+            'run_command': code + '\n',
+        }
+        self.check_config("init_run_main", core_config, preconfig)
+
+    def test_init_main(self):
         preconfig = {}
         code = ('import _testinternalcapi, json; '
                 'print(json.dumps(_testinternalcapi.get_configs()))')
@@ -698,8 +715,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'program': 'python3',
             'program_name': './python3',
             'run_command': code + '\n',
+            '_init_main': 0,
         }
-        self.check_config("run_main_config", core_config, preconfig)
+        self.check_config("init_main", core_config, preconfig,
+                          stderr="Run Python code before _Py_InitializeMain")
 
     def test_init_dont_parse_argv(self):
         core_config = {

Modules/getpath.c

@@ -493,7 +493,7 @@ calculate_prefix(const _PyCoreConfig *core_config,
     }
 
     if (!calculate->prefix_found) {
-        if (!core_config->_frozen) {
+        if (core_config->pathconfig_warnings) {
             fprintf(stderr,
                 "Could not find platform independent libraries <prefix>\n");
         }
@@ -681,7 +681,7 @@ calculate_exec_prefix(const _PyCoreConfig *core_config,
     }
 
     if (!calculate->exec_prefix_found) {
-        if (!core_config->_frozen) {
+        if (core_config->pathconfig_warnings) {
             fprintf(stderr,
                 "Could not find platform dependent libraries <exec_prefix>\n");
         }
@@ -1206,7 +1206,7 @@ calculate_path_impl(const _PyCoreConfig *core_config,
     }
 
     if ((!calculate->prefix_found || !calculate->exec_prefix_found) &&
-        !core_config->_frozen)
+        core_config->pathconfig_warnings)
     {
         fprintf(stderr,
                 "Consider setting $PYTHONHOME to <prefix>[:<exec_prefix>]\n");

Programs/_freeze_importlib.c

@@ -83,7 +83,8 @@ main(int argc, char *argv[])
     config.program_name = L"./_freeze_importlib";
     /* Don't install importlib, since it could execute outdated bytecode. */
     config._install_importlib = 0;
-    config._frozen = 1;
+    config.pathconfig_warnings = 0;
+    config._init_main = 0;
 
     _PyInitError err = _Py_InitializeFromConfig(&config);
     /* No need to call _PyCoreConfig_Clear() since we didn't allocate any

Programs/_testembed.c

@@ -757,34 +757,71 @@ fail:
 }
 
 
-static int test_run_main(void)
+wchar_t *init_main_argv[] = {
+    L"python3", L"-c",
+    (L"import _testinternalcapi, json; "
+     L"print(json.dumps(_testinternalcapi.get_configs()))"),
+    L"arg2"};
+
+
+static void configure_init_main(_PyCoreConfig *config)
+{
+    config->argv.length = Py_ARRAY_LENGTH(init_main_argv);
+    config->argv.items = init_main_argv;
+    config->program_name = L"./python3";
+}
+
+
+static int test_init_run_main(void)
 {
     _PyCoreConfig config = _PyCoreConfig_INIT;
+    configure_init_main(&config);
+
+    _PyInitError err = _Py_InitializeFromConfig(&config);
+    if (_Py_INIT_FAILED(err)) {
+        _Py_ExitInitError(err);
+    }
+
+    return _Py_RunMain();
+}
 
-    wchar_t *argv[] = {L"python3", L"-c",
-                       (L"import sys; "
-                        L"print(f'_Py_RunMain(): sys.argv={sys.argv}')"),
-                       L"arg2"};
-    config.argv.length = Py_ARRAY_LENGTH(argv);
-    config.argv.items = argv;
-    config.program_name = L"./python3";
+
+static int test_init_main(void)
+{
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+    configure_init_main(&config);
+    config._init_main = 0;
 
     _PyInitError err = _Py_InitializeFromConfig(&config);
     if (_Py_INIT_FAILED(err)) {
         _Py_ExitInitError(err);
     }
 
+    /* sys.stdout don't exist yet: it is created by _Py_InitializeMain() */
+    int res = PyRun_SimpleString(
+        "import sys; "
+        "print('Run Python code before _Py_InitializeMain', "
+               "file=sys.stderr)");
+    if (res < 0) {
+        exit(1);
+    }
+
+    err = _Py_InitializeMain();
+    if (_Py_INIT_FAILED(err)) {
+        _Py_ExitInitError(err);
+    }
+
     return _Py_RunMain();
 }
 
 
-static int test_run_main_config(void)
+static int test_run_main(void)
 {
     _PyCoreConfig config = _PyCoreConfig_INIT;
 
     wchar_t *argv[] = {L"python3", L"-c",
-                       (L"import _testinternalcapi, json; "
-                        L"print(json.dumps(_testinternalcapi.get_configs()))"),
+                       (L"import sys; "
+                        L"print(f'_Py_RunMain(): sys.argv={sys.argv}')"),
                        L"arg2"};
     config.argv.length = Py_ARRAY_LENGTH(argv);
     config.argv.items = argv;
@@ -837,8 +874,9 @@ static struct TestCase TestCases[] = {
     { "preinit_isolated1", test_preinit_isolated1 },
     { "preinit_isolated2", test_preinit_isolated2 },
     { "init_read_set", test_init_read_set },
+    { "init_run_main", test_init_run_main },
+    { "init_main", test_init_main },
     { "run_main", test_run_main },
-    { "run_main_config", test_run_main_config },
     { NULL, NULL }
 };
 

Python/coreconfig.c

@@ -667,7 +667,8 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
     COPY_WSTR_ATTR(run_module);
     COPY_WSTR_ATTR(run_filename);
     COPY_WSTR_ATTR(check_hash_pycs_mode);
-    COPY_ATTR(_frozen);
+    COPY_ATTR(pathconfig_warnings);
+    COPY_ATTR(_init_main);
 
 #undef COPY_ATTR
 #undef COPY_WSTR_ATTR
@@ -766,7 +767,8 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config)
     SET_ITEM_WSTR(run_filename);
     SET_ITEM_INT(_install_importlib);
     SET_ITEM_WSTR(check_hash_pycs_mode);
-    SET_ITEM_INT(_frozen);
+    SET_ITEM_INT(pathconfig_warnings);
+    SET_ITEM_INT(_init_main);
 
     return dict;
 
@@ -855,7 +857,7 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config)
 #ifdef MS_WINDOWS
     COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag);
 #endif
-    COPY_FLAG(_frozen, Py_FrozenFlag);
+    COPY_NOT_FLAG(pathconfig_warnings, Py_FrozenFlag);
 
     COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag);
     COPY_NOT_FLAG(site_import, Py_NoSiteFlag);
@@ -892,7 +894,7 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config)
 #ifdef MS_WINDOWS
     COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag);
 #endif
-    COPY_FLAG(_frozen, Py_FrozenFlag);
+    COPY_NOT_FLAG(pathconfig_warnings, Py_FrozenFlag);
 
     COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag);
     COPY_NOT_FLAG(site_import, Py_NoSiteFlag);
@@ -2253,7 +2255,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
     assert(!(config->run_command != NULL && config->run_module != NULL));
     assert(config->check_hash_pycs_mode != NULL);
     assert(config->_install_importlib >= 0);
-    assert(config->_frozen >= 0);
+    assert(config->pathconfig_warnings >= 0);
 
     err = _Py_INIT_OK();
 

Python/frozenmain.c

@@ -40,7 +40,7 @@ Py_FrozenMain(int argc, char **argv)
     }
 
     _PyCoreConfig config = _PyCoreConfig_INIT;
-    config._frozen = 1;   /* Suppress errors from getpath.c */
+    config.pathconfig_warnings = 0;   /* Suppress errors from getpath.c */
 
     if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0')
         inspect = 1;

Python/pylifecycle.c

@@ -970,6 +970,21 @@ _Py_InitializeMainInterpreter(_PyRuntimeState *runtime,
     return _Py_INIT_OK();
 }
 
+
+_PyInitError
+_Py_InitializeMain(void)
+{
+    _PyInitError err = _PyRuntime_Initialize();
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+    _PyRuntimeState *runtime = &_PyRuntime;
+    PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp;
+
+    return _Py_InitializeMainInterpreter(runtime, interp);
+}
+
+
 #undef _INIT_DEBUG_PRINT
 
 static _PyInitError
@@ -990,7 +1005,7 @@ init_python(const _PyCoreConfig *config, const _PyArgv *args)
     }
     config = &interp->core_config;
 
-    if (!config->_frozen) {
+    if (config->_init_main) {
         err = _Py_InitializeMainInterpreter(runtime, interp);
         if (_Py_INIT_FAILED(err)) {
             return err;

Python/pythonrun.c

@@ -1046,6 +1046,15 @@ run_eval_code_obj(PyCodeObject *co, PyObject *globals, PyObject *locals)
      * Py_Main() based one.
      */
     _Py_UnhandledKeyboardInterrupt = 0;
+
+    /* Set globals['__builtins__'] if it doesn't exist */
+    if (globals != NULL && PyDict_GetItemString(globals, "__builtins__") == NULL) {
+        PyInterpreterState *interp = _PyInterpreterState_Get();
+        if (PyDict_SetItemString(globals, "__builtins__", interp->builtins) < 0) {
+            return NULL;
+        }
+    }
+
     v = PyEval_EvalCode((PyObject*)co, globals, locals);
     if (!v && PyErr_Occurred() == PyExc_KeyboardInterrupt) {
         _Py_UnhandledKeyboardInterrupt = 1;