bpo-36142: Add _PyPreConfig.utf8_mode (GH-12174)

* Move following fields from _PyCoreConfig to _PyPreConfig:

  * coerce_c_locale
  * coerce_c_locale_warn
  * legacy_windows_stdio
  * utf8_mode

* _PyPreConfig_ReadFromArgv() is now responsible to choose the
  filesystem encoding
* _PyPreConfig_Write() now sets the LC_CTYPE locale

Include/cpython/coreconfig.h

@@ -60,12 +60,42 @@ typedef struct {
        Set to 0 by -E command line option. If set to -1 (default), it is
        set to !Py_IgnoreEnvironmentFlag. */
     int use_environment;
+
+    int coerce_c_locale;    /* PYTHONCOERCECLOCALE, -1 means unknown */
+    int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
+
+#ifdef MS_WINDOWS
+    /* If greater than 1, use the "mbcs" encoding instead of the UTF-8
+       encoding for the filesystem encoding.
+
+       Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is
+       set to a non-empty string. If set to -1 (default), inherit
+       Py_LegacyWindowsFSEncodingFlag value.
+
+       See PEP 529 for more details. */
+    int legacy_windows_fs_encoding;
+#endif
+
+    /* Enable UTF-8 mode?
+       Set by -X utf8 command line option and PYTHONUTF8 environment variable.
+       If set to -1 (default), inherit Py_UTF8Mode value. */
+    int utf8_mode;
 } _PyPreConfig;
 
+#ifdef MS_WINDOWS
+#  define _PyPreConfig_WINDOWS_INIT \
+        .legacy_windows_fs_encoding = -1,
+#else
+#  define _PyPreConfig_WINDOWS_INIT
+#endif
+
 #define _PyPreConfig_INIT \
     (_PyPreConfig){ \
+        _PyPreConfig_WINDOWS_INIT \
         .isolated = -1, \
-        .use_environment = -1}
+        .use_environment = -1, \
+        .coerce_c_locale = -1, \
+        .utf8_mode = -1}
 
 
 /* --- _PyCoreConfig ---------------------------------------------- */
@@ -95,8 +125,6 @@ typedef struct {
     int show_alloc_count;   /* -X showalloccount */
     int dump_refs;          /* PYTHONDUMPREFS */
     int malloc_stats;       /* PYTHONMALLOCSTATS */
-    int coerce_c_locale;    /* PYTHONCOERCECLOCALE, -1 means unknown */
-    int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
 
     /* Python filesystem encoding and error handler:
        sys.getfilesystemencoding() and sys.getfilesystemencodeerrors().
@@ -134,11 +162,6 @@ typedef struct {
     char *filesystem_encoding;
     char *filesystem_errors;
 
-    /* Enable UTF-8 mode?
-       Set by -X utf8 command line option and PYTHONUTF8 environment variable.
-       If set to -1 (default), inherit Py_UTF8Mode value. */
-    int utf8_mode;
-
     wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
 
     wchar_t *program_name;  /* Program name, see also Py_GetProgramName() */
@@ -277,16 +300,6 @@ typedef struct {
     char *stdio_errors;
 
 #ifdef MS_WINDOWS
-    /* If greater than 1, use the "mbcs" encoding instead of the UTF-8
-       encoding for the filesystem encoding.
-
-       Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is
-       set to a non-empty string. If set to -1 (default), inherit
-       Py_LegacyWindowsFSEncodingFlag value.
-
-       See PEP 529 for more details. */
-    int legacy_windows_fs_encoding;
-
     /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys
        standard streams.
 
@@ -340,7 +353,6 @@ typedef struct {
 
 #ifdef MS_WINDOWS
 #  define _PyCoreConfig_WINDOWS_INIT \
-        .legacy_windows_fs_encoding = -1, \
         .legacy_windows_stdio = -1,
 #else
 #  define _PyCoreConfig_WINDOWS_INIT
@@ -348,13 +360,12 @@ typedef struct {
 
 #define _PyCoreConfig_INIT \
     (_PyCoreConfig){ \
+        _PyCoreConfig_WINDOWS_INIT \
         .preconfig = _PyPreConfig_INIT, \
         .install_signal_handlers = 1, \
         .use_hash_seed = -1, \
         .faulthandler = -1, \
         .tracemalloc = -1, \
-        .coerce_c_locale = -1, \
-        .utf8_mode = -1, \
         .argc = -1, \
         .nmodule_search_path = -1, \
         .site_import = -1, \
@@ -368,7 +379,6 @@ typedef struct {
         .quiet = -1, \
         .user_site_directory = -1, \
         .buffered_stdio = -1, \
-        _PyCoreConfig_WINDOWS_INIT \
         ._install_importlib = 1, \
         ._check_hash_pycs_mode = "default", \
         ._frozen = -1}

Include/internal/pycore_coreconfig.h

@@ -36,11 +36,24 @@ PyAPI_FUNC(int) _Py_SetArgcArgv(int argc, wchar_t * const *argv);
 
 /* --- _PyPreConfig ----------------------------------------------- */
 
+PyAPI_FUNC(int) _Py_str_to_int(
+    const char *str,
+    int *result);
+PyAPI_FUNC(const wchar_t*) _Py_get_xoption(
+    int nxoption,
+    wchar_t * const *xoptions,
+    const wchar_t *name);
+
 PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config);
 PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config,
     const _PyPreConfig *config2);
 PyAPI_FUNC(void) _PyPreConfig_GetGlobalConfig(_PyPreConfig *config);
 PyAPI_FUNC(void) _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config);
+PyAPI_FUNC(const char*) _PyPreConfig_GetEnv(const _PyPreConfig *config,
+    const char *name);
+PyAPI_FUNC(void) _Py_get_env_flag(_PyPreConfig *config,
+    int *flag,
+    const char *name);
 PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config);
 PyAPI_FUNC(int) _PyPreConfig_AsDict(const _PyPreConfig *config,
     PyObject *dict);

Programs/_testembed.c

@@ -461,7 +461,7 @@ static int test_init_from_config(void)
 
     putenv("PYTHONUTF8=0");
     Py_UTF8Mode = 0;
-    config.utf8_mode = 1;
+    config.preconfig.utf8_mode = 1;
 
     putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix");
     config.pycache_prefix = L"conf_pycache_prefix";
@@ -610,8 +610,8 @@ static int test_init_isolated(void)
     config.preconfig.isolated = 1;
 
     /* Set coerce_c_locale and utf8_mode to not depend on the locale */
-    config.coerce_c_locale = 0;
-    config.utf8_mode = 0;
+    config.preconfig.coerce_c_locale = 0;
+    config.preconfig.utf8_mode = 0;
     /* Use path starting with "./" avoids a search along the PATH */
     config.program_name = L"./_testembed";
 

Python/pylifecycle.c

@@ -287,7 +287,7 @@ static const char *_C_LOCALE_WARNING =
 static void
 _emit_stderr_warning_for_legacy_locale(const _PyCoreConfig *core_config)
 {
-    if (core_config->coerce_c_locale_warn && _Py_LegacyLocaleDetected()) {
+    if (core_config->preconfig.coerce_c_locale_warn && _Py_LegacyLocaleDetected()) {
         PySys_FormatStderr("%s", _C_LOCALE_WARNING);
     }
 }

Python/sysmodule.c

@@ -2181,7 +2181,7 @@ make_flags(void)
     SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0);
     SetFlag(config->preconfig.isolated);
     PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode));
-    SetFlag(config->utf8_mode);
+    SetFlag(config->preconfig.utf8_mode);
 #undef SetFlag
 
     if (PyErr_Occurred()) {