bpo-32030: Fix usage of memory allocators (#4953)

* _Py_InitializeCore() doesn't call _PyMem_SetupAllocators() anymore
  if the PYTHONMALLOC environment variable is not set.
* pymain_cmdline() now sets the allocator to the default, instead of
  setting the allocator in subfunctions.
* Py_SetStandardStreamEncoding() now calls
  _PyMem_SetDefaultAllocator() to get a known allocator, to be able
  to release the memory with the same allocator.

Misc/NEWS.d/next/C API/2017-12-20-23-22-32.bpo-32030.d1dcwh.rst

+Py_Initialize() doesn't reset the memory allocators to default if the
+``PYTHONMALLOC`` environment variable is not set.

Modules/main.c

@@ -649,11 +649,12 @@ pymain_free_raw(_PyMain *pymain)
        Py_Initialize()-Py_Finalize() can be called multiple times. */
     _PyPathConfig_Clear(&_Py_path_config);
 
+    pymain_clear_config(pymain);
+
     /* Force the allocator used by pymain_read_conf() */
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    _PyCoreConfig_Clear(&pymain->config);
     pymain_clear_pymain(pymain);
 
     clear_wstrlist(orig_argc, orig_argv);
@@ -1963,11 +1964,6 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
 {
     int res = -1;
 
-    /* Force default allocator, since pymain_free() must use the same allocator
-       than this function. */
-    PyMemAllocatorEx old_alloc;
-    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
-
     char *oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
     if (oldloc == NULL) {
         pymain->err = _Py_INIT_NO_MEMORY();
@@ -2055,7 +2051,6 @@ done:
         PyMem_RawFree(oldloc);
     }
 
-    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
     return res;
 }
 
@@ -2578,6 +2573,15 @@ pymain_cmdline_impl(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
 static int
 pymain_cmdline(_PyMain *pymain)
 {
+    /* Force default allocator, since pymain_free() and pymain_clear_config()
+       must use the same allocator than this function. */
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+#ifdef Py_DEBUG
+    PyMemAllocatorEx default_alloc;
+    PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &default_alloc);
+#endif
+
     _Py_CommandLineDetails cmdline;
     memset(&cmdline, 0, sizeof(cmdline));
 
@@ -2588,6 +2592,14 @@ pymain_cmdline(_PyMain *pymain)
     pymain_set_global_config(pymain, &cmdline);
 
     pymain_clear_cmdline(pymain, &cmdline);
+
+#ifdef Py_DEBUG
+    /* Make sure that PYMEM_DOMAIN_RAW has not been modified */
+    PyMemAllocatorEx cur_alloc;
+    PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &cur_alloc);
+    assert(memcmp(&cur_alloc, &default_alloc, sizeof(cur_alloc)) == 0);
+#endif
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
     return res;
 }
 

Python/pylifecycle.c

@@ -172,6 +172,15 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
         /* This is too late to have any effect */
         return -1;
     }
+
+    int res = 0;
+
+    /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(),
+       but Py_Initialize() can change the allocator. Use a known allocator
+       to be able to release the memory later. */
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
     /* Can't call PyErr_NoMemory() on errors, as Python hasn't been
      * initialised yet.
      *
@@ -182,7 +191,8 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
     if (encoding) {
         _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding);
         if (!_Py_StandardStreamEncoding) {
-            return -2;
+            res = -2;
+            goto done;
         }
     }
     if (errors) {
@@ -191,7 +201,8 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
             if (_Py_StandardStreamEncoding) {
                 PyMem_RawFree(_Py_StandardStreamEncoding);
             }
-            return -3;
+            res = -3;
+            goto done;
         }
     }
 #ifdef MS_WINDOWS
@@ -200,7 +211,11 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
         Py_LegacyWindowsStdioFlag = 1;
     }
 #endif
-    return 0;
+
+done:
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
+    return res;
 }
 
 
@@ -597,8 +612,10 @@ _Py_InitializeCore(const _PyCoreConfig *core_config)
         return err;
     }
 
-    if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
-        return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
+    if (core_config->allocator != NULL) {
+        if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
+            return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
+        }
     }
 
     if (_PyRuntime.initialized) {
@@ -1818,7 +1835,11 @@ init_sys_streams(PyInterpreterState *interp)
 error:
     res = _Py_INIT_ERR("can't initialize sys standard streams");
 
+    /* Use the same allocator than Py_SetStandardStreamEncoding() */
+    PyMemAllocatorEx old_alloc;
 done:
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
     /* We won't need them anymore. */
     if (_Py_StandardStreamEncoding) {
         PyMem_RawFree(_Py_StandardStreamEncoding);
@@ -1828,6 +1849,9 @@ done:
         PyMem_RawFree(_Py_StandardStreamErrors);
         _Py_StandardStreamErrors = NULL;
     }
+
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+
     PyMem_Free(pythonioencoding);
     Py_XDECREF(bimod);
     Py_XDECREF(iomod);