bpo-34589: C locale coercion off by default (GH-9073)

Py_Initialize() and Py_Main() cannot enable the C locale coercion
(PEP 538) anymore: it is always disabled. It can now only be enabled
by the Python program ("python3).

test_embed: get_filesystem_encoding() doesn't have to set PYTHONUTF8
nor PYTHONCOERCECLOCALE, these variables are already set in the
parent.

Include/coreconfig.h

@@ -301,6 +301,10 @@ typedef struct {
        variable. The option is also enabled if the LC_CTYPE locale is "C"
        and a target locale (ex: "C.UTF-8") is supported by the platform.
 
+       Py_Initialize() and Py_Main() must not enable C locale coercion: it is
+       always disabled. The option can only be enabled by the Python program
+       ("python3).
+
        See also the _coerce_c_locale_warn option. */
     int _coerce_c_locale;
 
@@ -308,6 +312,10 @@ typedef struct {
 
        Enabled by the PYTHONCOERCECLOCALE=warn environment variable.
 
+       Py_Initialize() and Py_Main() must not enable C locale coercion warning:
+       it is always disabled. The warning can only be enabled by the Python
+       program ("python3).
+
        See also the _coerce_c_locale option. */
     int _coerce_c_locale_warn;
 
@@ -328,7 +336,8 @@ typedef struct {
         .use_hash_seed = -1, \
         .faulthandler = -1, \
         .tracemalloc = -1, \
-        ._coerce_c_locale = -1, \
+        ._coerce_c_locale = 0, \
+        ._coerce_c_locale_warn = 0, \
         .utf8_mode = -1, \
         .argc = -1, \
         .nmodule_search_path = -1, \

Include/pylifecycle.h

@@ -83,7 +83,11 @@ PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
 /* Bootstrap __main__ (defined in Modules/main.c) */
 PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv);
 #ifdef Py_BUILD_CORE
+#  ifdef MS_WINDOWS
+PyAPI_FUNC(int) _Py_WindowsMain(int argc, wchar_t **argv);
+#  else
 PyAPI_FUNC(int) _Py_UnixMain(int argc, char **argv);
+#  endif
 #endif
 
 /* In getpath.c */

Lib/test/test_embed.py

@@ -324,10 +324,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 'print(sys.getfilesystemencoding(), '
                 'sys.getfilesystemencodeerrors())')
         args = (sys.executable, '-c', code)
-        env = dict(env)
-        if not isolated:
-            env['PYTHONCOERCECLOCALE'] = '0'
-            env['PYTHONUTF8'] = '0'
         proc = subprocess.run(args, text=True, env=env,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)

Misc/NEWS.d/next/C API/2018-09-18-00-09-31.bpo-34589.C7bUpq.rst

+Py_Initialize() and Py_Main() cannot enable the C locale coercion (PEP 538)
+anymore: it is always disabled. It can now only be enabled by the Python
+program ("python3).

Modules/main.c

@@ -1700,7 +1700,8 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config)
 
 
 static int
-pymain_init(_PyMain *pymain, PyInterpreterState **interp_p)
+pymain_init(_PyMain *pymain, PyInterpreterState **interp_p,
+            int use_c_locale_coercion)
 {
     /* 754 requires that FP exceptions run in "no stop" mode by default,
      * and until C vendors implement C99's ways to control FP exceptions,
@@ -1713,6 +1714,11 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p)
 
     _PyCoreConfig local_config = _PyCoreConfig_INIT;
     _PyCoreConfig *config = &local_config;
+    if (use_c_locale_coercion) {
+        /* set to -1 to be able to enable the feature */
+        config->_coerce_c_locale = -1;
+        config->_coerce_c_locale_warn = -1;
+    }
 
     _PyCoreConfig_GetGlobalConfig(config);
 
@@ -1747,10 +1753,10 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p)
 
 
 static int
-pymain_main(_PyMain *pymain)
+pymain_main(_PyMain *pymain, int use_c_locale_coercion)
 {
     PyInterpreterState *interp;
-    int res = pymain_init(pymain, &interp);
+    int res = pymain_init(pymain, &interp, use_c_locale_coercion);
     if (res != 1) {
         if (pymain_run_python(pymain, interp) < 0) {
             _Py_FatalInitError(pymain->err);
@@ -1777,10 +1783,22 @@ Py_Main(int argc, wchar_t **argv)
     pymain.argc = argc;
     pymain.wchar_argv = argv;
 
-    return pymain_main(&pymain);
+    return pymain_main(&pymain, 0);
 }
 
 
+#ifdef MS_WINDOWS
+int
+_Py_WindowsMain(int argc, wchar_t **argv)
+{
+    _PyMain pymain = _PyMain_INIT;
+    pymain.use_bytes_argv = 0;
+    pymain.argc = argc;
+    pymain.wchar_argv = argv;
+
+    return pymain_main(&pymain, 1);
+}
+#else
 int
 _Py_UnixMain(int argc, char **argv)
 {
@@ -1789,8 +1807,9 @@ _Py_UnixMain(int argc, char **argv)
     pymain.argc = argc;
     pymain.bytes_argv = argv;
 
-    return pymain_main(&pymain);
+    return pymain_main(&pymain, 1);
 }
+#endif
 
 
 /* this is gonna seem *real weird*, but if you put some other code between

Programs/_testembed.c

@@ -482,8 +482,6 @@ static int test_init_from_config(void)
     putenv("PYTHONMALLOCSTATS=0");
     config.malloc_stats = 1;
 
-    /* FIXME: test _coerce_c_locale and _coerce_c_locale_warn */
-
     putenv("PYTHONUTF8=0");
     Py_UTF8Mode = 0;
     config.utf8_mode = 1;
@@ -606,8 +604,7 @@ static int test_init_isolated(void)
     /* Test _PyCoreConfig.isolated=1 */
     _PyCoreConfig config = _PyCoreConfig_INIT;
 
-    /* Set _coerce_c_locale and utf8_mode to not depend on the locale */
-    config._coerce_c_locale = 0;
+    /* Set utf8_mode to not depend on the locale */
     config.utf8_mode = 0;
     /* Use path starting with "./" avoids a search along the PATH */
     config.program_name = L"./_testembed";

Programs/python.c

@@ -6,7 +6,7 @@
 int
 wmain(int argc, wchar_t **argv)
 {
-    return Py_Main(argc, argv);
+    return _Py_WindowsMain(argc, argv);
 }
 #else
 int

Python/coreconfig.c

@@ -816,8 +816,10 @@ config_read_env_vars(_PyCoreConfig *config)
             }
         }
         else if (strcmp(env, "warn") == 0) {
+            if (config->_coerce_c_locale_warn < 0) {
                 config->_coerce_c_locale_warn = 1;
             }
+        }
         else {
             if (config->_coerce_c_locale < 0) {
                 config->_coerce_c_locale = 1;
@@ -1324,6 +1326,9 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
     if (config->_coerce_c_locale < 0) {
         config->_coerce_c_locale = 0;
     }
+    if (config->_coerce_c_locale_warn < 0) {
+        config->_coerce_c_locale_warn = 0;
+    }
     if (config->utf8_mode < 0) {
         config->utf8_mode = 0;
     }