bpo-36763: Add _PyCoreConfig.parse_argv (GH-13361)

* _PyCoreConfig_Read() doesn't parse nor update argv
  if parse_argv is 0.
* Move path configuration fields in _PyCoreConfig.
* Add an unit test for parse_argv=0.
* Remove unused "done": label in _Py_RunMain().

Include/cpython/coreconfig.h

@@ -207,31 +207,25 @@ typedef struct {
     wchar_t *filesystem_errors;
 
     wchar_t *pycache_prefix;  /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
-    wchar_t *program_name;    /* Program name, see also Py_GetProgramName() */
-    _PyWstrList argv;         /* Command line arguments */
-    wchar_t *program;         /* argv[0] or "" */
-    _PyWstrList xoptions;     /* Command line -X options */
-    _PyWstrList warnoptions;  /* Warnings options */
+    int parse_argv;           /* Parse argv command line arguments? */
 
-    /* Path configuration inputs */
-    wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
-    wchar_t *home;          /* PYTHONHOME environment variable,
-                               see also Py_SetPythonHome(). */
+    /* Command line arguments (sys.argv).
 
-    /* Path configuration outputs */
-    int use_module_search_paths;  /* If non-zero, use module_search_paths */
-    _PyWstrList module_search_paths;  /* sys.path paths. Computed if
-                                       use_module_search_paths is equal
-                                       to zero. */
+       By default, Python command line arguments are parsed and then stripped
+       from argv. Set parse_argv to 0 to avoid that.
 
-    wchar_t *executable;    /* sys.executable */
-    wchar_t *prefix;        /* sys.prefix */
-    wchar_t *base_prefix;   /* sys.base_prefix */
-    wchar_t *exec_prefix;   /* sys.exec_prefix */
-    wchar_t *base_exec_prefix;  /* sys.base_exec_prefix */
-#ifdef MS_WINDOWS
-    wchar_t *dll_path;      /* Windows DLL path */
-#endif
+       If argv is empty, an empty string is added to ensure that sys.argv
+       always exists and is never empty. */
+    _PyWstrList argv;
+
+    /* Program: argv[0] or "".
+       Used to display Python usage if parsing command line arguments fails.
+       Used to initialize the default value of program_name */
+    wchar_t *program;
+    wchar_t *program_name;    /* Program name, see also Py_GetProgramName() */
+
+    _PyWstrList xoptions;     /* Command line -X options */
+    _PyWstrList warnoptions;  /* Warnings options */
 
     /* If equal to zero, disable the import of the module site and the
        site-dependent manipulations of sys.path that it entails. Also disable
@@ -350,6 +344,28 @@ typedef struct {
     int legacy_windows_stdio;
 #endif
 
+    /* --- Path configuration inputs ------------ */
+
+    wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
+    wchar_t *home;          /* PYTHONHOME environment variable,
+                               see also Py_SetPythonHome(). */
+
+    /* --- Path configuration outputs ----------- */
+
+    int use_module_search_paths;  /* If non-zero, use module_search_paths */
+    _PyWstrList module_search_paths;  /* sys.path paths. Computed if
+                                       use_module_search_paths is equal
+                                       to zero. */
+
+    wchar_t *executable;    /* sys.executable */
+    wchar_t *prefix;        /* sys.prefix */
+    wchar_t *base_prefix;   /* sys.base_prefix */
+    wchar_t *exec_prefix;   /* sys.exec_prefix */
+    wchar_t *base_exec_prefix;  /* sys.base_exec_prefix */
+#ifdef MS_WINDOWS
+    wchar_t *dll_path;      /* Windows DLL path */
+#endif
+
     /* --- Parameter only used by Py_Main() ---------- */
 
     /* Skip the first line of the source ('run_filename' parameter), allowing use of non-Unix forms of
@@ -408,6 +424,7 @@ typedef struct {
         .faulthandler = -1, \
         .tracemalloc = -1, \
         .use_module_search_paths = 0, \
+        .parse_argv = 1, \
         .site_import = -1, \
         .bytes_warning = -1, \
         .inspect = -1, \

Lib/test/test_embed.py

@@ -304,6 +304,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         'pycache_prefix': None,
         'program_name': GET_DEFAULT_CONFIG,
+        'parse_argv': 1,
         'argv': [""],
         'program': '',
 
@@ -700,6 +701,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         }
         self.check_config("run_main_config", core_config, preconfig)
 
+    def test_init_dont_parse_argv(self):
+        core_config = {
+            'argv': ['-v', '-c', 'arg1', '-W', 'arg2'],
+            'parse_argv': 0,
+            'program': 'program',
+        }
+        self.check_config("init_dont_parse_argv", core_config, {})
+
 
 if __name__ == "__main__":
     unittest.main()

Modules/main.c

@@ -574,13 +574,13 @@ _Py_RunMain(void)
     int exitcode = 0;
 
     pymain_run_python(&exitcode);
+
     if (Py_FinalizeEx() < 0) {
         /* Value unlikely to be confused with a non-error exit status or
            other special meaning */
         exitcode = 120;
     }
 
-done:
     pymain_free();
 
     if (_Py_UnhandledKeyboardInterrupt) {

Programs/_testembed.c

@@ -511,6 +511,37 @@ static int test_init_from_config(void)
 }
 
 
+static int test_init_dont_parse_argv(void)
+{
+    _PyInitError err;
+
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+
+    static wchar_t* argv[] = {
+        L"-v",
+        L"-c",
+        L"arg1",
+        L"-W",
+        L"arg2",
+    };
+
+    config.program = L"program";
+    config.program_name = L"./_testembed";
+
+    config.argv.length = Py_ARRAY_LENGTH(argv);
+    config.argv.items = argv;
+    config.parse_argv = 0;
+
+    err = _Py_InitializeFromConfig(&config);
+    if (_Py_INIT_FAILED(err)) {
+        _Py_ExitInitError(err);
+    }
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static void test_init_env_putenvs(void)
 {
     putenv("PYTHONHASHSEED=42");
@@ -797,6 +828,7 @@ static struct TestCase TestCases[] = {
     { "init_default_config", test_init_default_config },
     { "init_global_config", test_init_global_config },
     { "init_from_config", test_init_from_config },
+    { "init_dont_parse_argv", test_init_dont_parse_argv },
     { "init_env", test_init_env },
     { "init_env_dev_mode", test_init_env_dev_mode },
     { "init_env_dev_mode_alloc", test_init_env_dev_mode_alloc },

Python/coreconfig.c

@@ -628,6 +628,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
     COPY_WSTR_ATTR(program_name);
     COPY_WSTR_ATTR(program);
 
+    COPY_ATTR(parse_argv);
     COPY_WSTRLIST(argv);
     COPY_WSTRLIST(warnoptions);
     COPY_WSTRLIST(xoptions);
@@ -727,6 +728,7 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config)
     SET_ITEM_WSTR(filesystem_errors);
     SET_ITEM_WSTR(pycache_prefix);
     SET_ITEM_WSTR(program_name);
+    SET_ITEM_INT(parse_argv);
     SET_ITEM_WSTRLIST(argv);
     SET_ITEM_WSTR(program);
     SET_ITEM_WSTRLIST(xoptions);
@@ -1490,6 +1492,8 @@ config_read(_PyCoreConfig *config, _PyPreCmdline *cmdline)
     }
 
     if (config->isolated > 0) {
+        /* _PyPreCmdline_Read() sets use_environment to 0 if isolated is set,
+           _PyPreCmdline_SetCoreConfig() overrides config->use_environment. */
         config->user_site_directory = 0;
     }
 
@@ -1660,7 +1664,7 @@ config_usage(int error, const wchar_t* program)
 /* Parse the command line arguments */
 static _PyInitError
 config_parse_cmdline(_PyCoreConfig *config, _PyPreCmdline *precmdline,
-                     _PyWstrList *warnoptions)
+                     _PyWstrList *warnoptions, int *opt_index)
 {
     _PyInitError err;
     const _PyWstrList *argv = &precmdline->argv;
@@ -1833,8 +1837,7 @@ config_parse_cmdline(_PyCoreConfig *config, _PyPreCmdline *precmdline,
         _PyOS_optind--;
     }
 
-    /* -c and -m options are exclusive */
-    assert(!(config->run_command != NULL && config->run_module != NULL));
+    *opt_index = _PyOS_optind;
 
     return _Py_INIT_OK();
 }
@@ -1978,13 +1981,14 @@ config_init_warnoptions(_PyCoreConfig *config,
 
 
 static _PyInitError
-config_init_argv(_PyCoreConfig *config, const _PyPreCmdline *cmdline)
+config_update_argv(_PyCoreConfig *config, const _PyPreCmdline *cmdline,
+                   int opt_index)
 {
     const _PyWstrList *cmdline_argv = &cmdline->argv;
     _PyWstrList config_argv = _PyWstrList_INIT;
 
     /* Copy argv to be able to modify it (to force -c/-m) */
-    if (cmdline_argv->length <= _PyOS_optind) {
+    if (cmdline_argv->length <= opt_index) {
         /* Ensure at least one (empty) argument is seen */
         if (_PyWstrList_Append(&config_argv, L"") < 0) {
             return _Py_INIT_NO_MEMORY();
@@ -1992,8 +1996,8 @@ config_init_argv(_PyCoreConfig *config, const _PyPreCmdline *cmdline)
     }
     else {
         _PyWstrList slice;
-        slice.length = cmdline_argv->length - _PyOS_optind;
-        slice.items = &cmdline_argv->items[_PyOS_optind];
+        slice.length = cmdline_argv->length - opt_index;
+        slice.items = &cmdline_argv->items[opt_index];
         if (_PyWstrList_Copy(&config_argv, &slice) < 0) {
             return _Py_INIT_NO_MEMORY();
         }
@@ -2058,14 +2062,22 @@ config_read_cmdline(_PyCoreConfig *config, _PyPreCmdline *precmdline)
     _PyWstrList cmdline_warnoptions = _PyWstrList_INIT;
     _PyWstrList env_warnoptions = _PyWstrList_INIT;
 
-    err = config_parse_cmdline(config, precmdline, &cmdline_warnoptions);
-    if (_Py_INIT_FAILED(err)) {
-        goto done;
+    if (config->parse_argv < 0) {
+        config->parse_argv = 1;
     }
 
-    err = config_init_argv(config, precmdline);
-    if (_Py_INIT_FAILED(err)) {
-        goto done;
+    if (config->parse_argv) {
+        int opt_index;
+        err = config_parse_cmdline(config, precmdline, &cmdline_warnoptions,
+                                   &opt_index);
+        if (_Py_INIT_FAILED(err)) {
+            goto done;
+        }
+
+        err = config_update_argv(config, precmdline, opt_index);
+        if (_Py_INIT_FAILED(err)) {
+            goto done;
+        }
     }
 
     err = config_read(config, precmdline);
@@ -2212,6 +2224,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
     assert(config->verbose >= 0);
     assert(config->quiet >= 0);
     assert(config->user_site_directory >= 0);
+    assert(config->parse_argv >= 0);
     assert(config->buffered_stdio >= 0);
     assert(config->program_name != NULL);
     assert(config->program != NULL);
@@ -2236,6 +2249,8 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
 #ifdef MS_WINDOWS
     assert(config->legacy_windows_stdio >= 0);
 #endif
+    /* -c and -m options are exclusive */
+    assert(!(config->run_command != NULL && config->run_module != NULL));
     assert(config->check_hash_pycs_mode != NULL);
     assert(config->_install_importlib >= 0);
     assert(config->_frozen >= 0);