bpo-36142: PYTHONMALLOC overrides PYTHONDEV (GH-12191)

bpo-34247, bpo-36142: The PYTHONMALLOC environment variable has the
priority over PYTHONDEV env var and "-X dev" command line option.
For example, PYTHONMALLOC=malloc PYTHONDEVMODE=1 sets the memory
allocators to "malloc" (and not to "debug").

Add an unit test.

Lib/test/test_embed.py

@@ -524,7 +524,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'install_signal_handlers': 0,
             'use_hash_seed': 1,
             'hash_seed': 123,
-            'allocator': 'malloc_debug',
+            'allocator': 'malloc',
             'tracemalloc': 2,
             'import_time': 1,
             'show_ref_count': 1,
@@ -564,7 +564,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
     INIT_ENV_CONFIG = {
         'use_hash_seed': 1,
         'hash_seed': 42,
-        'allocator': 'malloc_debug',
+        'allocator': 'malloc',
         'tracemalloc': 2,
         'import_time': 1,
         'malloc_stats': 1,
@@ -592,6 +592,12 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                       dev_mode=1)
         self.check_config("init_env_dev_mode", config)
 
+    def test_init_env_dev_mode(self):
+        config = dict(self.INIT_ENV_CONFIG,
+                      allocator='malloc',
+                      dev_mode=1)
+        self.check_config("init_env_dev_mode_alloc", config)
+
     def test_init_dev_mode(self):
         config = {
             'dev_mode': 1,

Programs/_testembed.c

@@ -442,8 +442,8 @@ static int test_init_from_config(void)
     config.use_hash_seed = 1;
     config.hash_seed = 123;
 
-    putenv("PYTHONMALLOC=malloc");
-    config.preconfig.allocator = "malloc_debug";
+    putenv("PYTHONMALLOC=malloc_debug");
+    config.preconfig.allocator = "malloc";
 
     /* dev_mode=1 is tested in test_init_dev_mode() */
 
@@ -570,7 +570,7 @@ static int test_init_from_config(void)
 static void test_init_env_putenvs(void)
 {
     putenv("PYTHONHASHSEED=42");
-    putenv("PYTHONMALLOC=malloc_debug");
+    putenv("PYTHONMALLOC=malloc");
     putenv("PYTHONTRACEMALLOC=2");
     putenv("PYTHONPROFILEIMPORTTIME=1");
     putenv("PYTHONMALLOCSTATS=1");
@@ -594,20 +594,32 @@ static void test_init_env_putenvs(void)
 }
 
 
+static int test_init_env(void)
+{
+    /* Test initialization from environment variables */
+    Py_IgnoreEnvironmentFlag = 0;
+    test_init_env_putenvs();
+    _testembed_Py_Initialize();
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static void test_init_env_dev_mode_putenvs(void)
 {
     test_init_env_putenvs();
-    putenv("PYTHONMALLOC=malloc");
+    putenv("PYTHONMALLOC=");
     putenv("PYTHONFAULTHANDLER=");
     putenv("PYTHONDEVMODE=1");
 }
 
 
-static int test_init_env(void)
+static int test_init_env_dev_mode(void)
 {
     /* Test initialization from environment variables */
     Py_IgnoreEnvironmentFlag = 0;
-    test_init_env_putenvs();
+    test_init_env_dev_mode_putenvs();
     _testembed_Py_Initialize();
     dump_config();
     Py_Finalize();
@@ -615,11 +627,12 @@ static int test_init_env(void)
 }
 
 
-static int test_init_env_dev_mode(void)
+static int test_init_env_dev_mode_alloc(void)
 {
     /* Test initialization from environment variables */
     Py_IgnoreEnvironmentFlag = 0;
     test_init_env_dev_mode_putenvs();
+    putenv("PYTHONMALLOC=malloc");
     _testembed_Py_Initialize();
     dump_config();
     Py_Finalize();
@@ -700,6 +713,7 @@ static struct TestCase TestCases[] = {
     { "init_from_config", test_init_from_config },
     { "init_env", test_init_env },
     { "init_env_dev_mode", test_init_env_dev_mode },
+    { "init_env_dev_mode_alloc", test_init_env_dev_mode_alloc },
     { "init_dev_mode", test_init_dev_mode },
     { "init_isolated", test_init_isolated },
     { NULL, NULL }

Python/preconfig.c

@@ -445,14 +445,11 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
     }
 
     /* allocator */
-    if (config->dev_mode && config->allocator == NULL) {
-        config->allocator = _PyMem_RawStrdup("debug");
-        if (config->allocator == NULL) {
-            return _Py_INIT_NO_MEMORY();
-        }
-    }
-
     if (config->allocator == NULL) {
+        /* bpo-34247. The PYTHONMALLOC environment variable has the priority
+           over PYTHONDEV env var and "-X dev" command line option.
+           For example, PYTHONMALLOC=malloc PYTHONDEVMODE=1 sets the memory
+           allocators to "malloc" (and not to "debug"). */
         const char *allocator = _PyPreConfig_GetEnv(config, "PYTHONMALLOC");
         if (allocator) {
             config->allocator = _PyMem_RawStrdup(allocator);
@@ -462,6 +459,13 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
         }
     }
 
+    if (config->dev_mode && config->allocator == NULL) {
+        config->allocator = _PyMem_RawStrdup("debug");
+        if (config->allocator == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
     assert(config->coerce_c_locale >= 0);
     assert(config->utf8_mode >= 0);
     assert(config->isolated >= 0);