diff -urN Python-2.7.10.ORIG/Include/objimpl.h Python-2.7.10/Include/objimpl.h --- Python-2.7.10.ORIG/Include/objimpl.h 2015-05-24 01:08:59.000000000 +0900 +++ Python-2.7.10/Include/objimpl.h 2015-10-13 17:31:13.771317208 +0900 @@ -98,10 +98,8 @@ PyAPI_FUNC(void *) PyObject_Realloc(void *, size_t); PyAPI_FUNC(void) PyObject_Free(void *); - /* Macros */ -#ifdef WITH_PYMALLOC -#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ +#if defined(WITH_PYMALLOC) && defined(PYMALLOC_DEBUG) PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes); PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes); PyAPI_FUNC(void) _PyObject_DebugFree(void *p); @@ -115,28 +113,17 @@ PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes); PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes); PyAPI_FUNC(void) _PyMem_DebugFree(void *p); -#define PyObject_MALLOC _PyObject_DebugMalloc -#define PyObject_Malloc _PyObject_DebugMalloc -#define PyObject_REALLOC _PyObject_DebugRealloc -#define PyObject_Realloc _PyObject_DebugRealloc -#define PyObject_FREE _PyObject_DebugFree -#define PyObject_Free _PyObject_DebugFree +#endif -#else /* WITH_PYMALLOC && ! PYMALLOC_DEBUG */ #define PyObject_MALLOC PyObject_Malloc #define PyObject_REALLOC PyObject_Realloc #define PyObject_FREE PyObject_Free -#endif - -#else /* ! WITH_PYMALLOC */ -#define PyObject_MALLOC PyMem_MALLOC -#define PyObject_REALLOC PyMem_REALLOC -#define PyObject_FREE PyMem_FREE - -#endif /* WITH_PYMALLOC */ - #define PyObject_Del PyObject_Free -#define PyObject_DEL PyObject_FREE +#define PyObject_DEL PyObject_Free + +#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ +PyAPI_FUNC(void) _PyObject_DebugMallocStats(void); +#endif /* for source compatibility with 2.2 */ #define _PyObject_Del PyObject_Free diff -urN Python-2.7.10.ORIG/Include/pymem.h Python-2.7.10/Include/pymem.h --- Python-2.7.10.ORIG/Include/pymem.h 2015-05-24 01:09:00.000000000 +0900 +++ Python-2.7.10/Include/pymem.h 2015-10-13 17:31:13.771317208 +0900 @@ -11,6 +11,11 @@ extern "C" { #endif +PyAPI_FUNC(void *) PyMem_RawMalloc(size_t size); +PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); +PyAPI_FUNC(void) PyMem_RawFree(void *ptr); + + /* BEWARE: Each interface exports both functions and macros. Extension modules should @@ -49,21 +54,17 @@ performed on failure (no exception is set, no warning is printed, etc). */ -PyAPI_FUNC(void *) PyMem_Malloc(size_t); -PyAPI_FUNC(void *) PyMem_Realloc(void *, size_t); -PyAPI_FUNC(void) PyMem_Free(void *); +PyAPI_FUNC(void *) PyMem_Malloc(size_t size); +PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size); +PyAPI_FUNC(void) PyMem_Free(void *ptr); + +PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); +PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); /* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are no longer supported. They used to call PyErr_NoMemory() on failure. */ /* Macros. */ -#ifdef PYMALLOC_DEBUG -/* Redirect all memory operations to Python's debugging allocator. */ -#define PyMem_MALLOC _PyMem_DebugMalloc -#define PyMem_REALLOC _PyMem_DebugRealloc -#define PyMem_FREE _PyMem_DebugFree - -#else /* ! PYMALLOC_DEBUG */ /* PyMem_MALLOC(0) means malloc(1). Some systems would return NULL for malloc(0), which would be treated as an error. Some platforms @@ -71,13 +72,9 @@ pymalloc. To solve these problems, allocate an extra byte. */ /* Returns NULL to indicate error if a negative size or size larger than Py_ssize_t can represent is supplied. Helps prevents security holes. */ -#define PyMem_MALLOC(n) ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \ - : malloc((n) ? (n) : 1)) -#define PyMem_REALLOC(p, n) ((size_t)(n) > (size_t)PY_SSIZE_T_MAX ? NULL \ - : realloc((p), (n) ? (n) : 1)) -#define PyMem_FREE free - -#endif /* PYMALLOC_DEBUG */ +#define PyMem_MALLOC(n) PyMem_Malloc(n) +#define PyMem_REALLOC(p, n) PyMem_Realloc(p, n) +#define PyMem_FREE(p) PyMem_Free(p) /* * Type-oriented memory interface @@ -115,6 +112,67 @@ #define PyMem_Del PyMem_Free #define PyMem_DEL PyMem_FREE +typedef enum { + /* PyMem_RawMalloc(), PyMem_RawRealloc() and PyMem_RawFree() */ + PYMEM_DOMAIN_RAW, + + /* PyMem_Malloc(), PyMem_Realloc() and PyMem_Free() */ + PYMEM_DOMAIN_MEM, + + /* PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() */ + PYMEM_DOMAIN_OBJ +} PyMemAllocatorDomain; + +typedef struct { + /* user context passed as the first argument to the 3 functions */ + void *ctx; + + /* allocate a memory block */ + void* (*malloc) (void *ctx, size_t size); + + /* allocate or resize a memory block */ + void* (*realloc) (void *ctx, void *ptr, size_t new_size); + + /* release a memory block */ + void (*free) (void *ctx, void *ptr); +} PyMemAllocator; + +/* Get the memory block allocator of the specified domain. */ +PyAPI_FUNC(void) PyMem_GetAllocator(PyMemAllocatorDomain domain, + PyMemAllocator *allocator); + +/* Set the memory block allocator of the specified domain. + + The new allocator must return a distinct non-NULL pointer when requesting + zero bytes. + + For the PYMEM_DOMAIN_RAW domain, the allocator must be thread-safe: the GIL + is not held when the allocator is called. + + If the new allocator is not a hook (don't call the previous allocator), the + PyMem_SetupDebugHooks() function must be called to reinstall the debug hooks + on top on the new allocator. */ +PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain, + PyMemAllocator *allocator); + +/* Setup hooks to detect bugs in the following Python memory allocator + functions: + + - PyMem_RawMalloc(), PyMem_RawRealloc(), PyMem_RawFree() + - PyMem_Malloc(), PyMem_Realloc(), PyMem_Free() + - PyObject_Malloc(), PyObject_Realloc() and PyObject_Free() + + Newly allocated memory is filled with the byte 0xCB, freed memory is filled + with the byte 0xDB. Additionnal checks: + + - detect API violations, ex: PyObject_Free() called on a buffer allocated + by PyMem_Malloc() + - detect write before the start of the buffer (buffer underflow) + - detect write after the end of the buffer (buffer overflow) + + The function does nothing if Python is not compiled is debug mode. */ +PyAPI_FUNC(void) PyMem_SetupDebugHooks(void); + #ifdef __cplusplus } #endif diff -urN Python-2.7.10.ORIG/Objects/object.c Python-2.7.10/Objects/object.c --- Python-2.7.10.ORIG/Objects/object.c 2015-05-24 01:09:22.000000000 +0900 +++ Python-2.7.10/Objects/object.c 2015-10-13 17:31:13.771317208 +0900 @@ -2335,27 +2335,6 @@ Py_ssize_t (*_Py_abstract_hack)(PyObject *) = PyObject_Size; -/* Python's malloc wrappers (see pymem.h) */ - -void * -PyMem_Malloc(size_t nbytes) -{ - return PyMem_MALLOC(nbytes); -} - -void * -PyMem_Realloc(void *p, size_t nbytes) -{ - return PyMem_REALLOC(p, nbytes); -} - -void -PyMem_Free(void *p) -{ - PyMem_FREE(p); -} - - /* These methods are used to control infinite recursion in repr, str, print, etc. Container objects that may recursively contain themselves, e.g. builtin dictionaries and lists, should used Py_ReprEnter() and diff -urN Python-2.7.10.ORIG/Objects/obmalloc.c Python-2.7.10/Objects/obmalloc.c --- Python-2.7.10.ORIG/Objects/obmalloc.c 2015-05-24 01:09:22.000000000 +0900 +++ Python-2.7.10/Objects/obmalloc.c 2015-10-13 17:37:24.665726972 +0900 @@ -18,6 +18,281 @@ #endif #endif +/* Python's malloc wrappers (see pymem.h) */ + +#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ +/* Forward declaration */ +static void* _PyMem_DebugMallocCtx(void *ctx, size_t size); +static void _PyMem_DebugFreeCtx(void *ctx, void *p); +static void* _PyMem_DebugReallocCtx(void *ctx, void *ptr, size_t size); + +static void _PyMem_DebugCheckAddress(char api_id, const void *p); +#endif + +#ifdef WITH_PYMALLOC + +#ifdef MS_WINDOWS +# include <windows.h> +#elif defined(HAVE_MMAP) +# include <sys/mman.h> +# ifdef MAP_ANONYMOUS +# define ARENAS_USE_MMAP +# endif +#endif + +/* Forward declaration */ +static void* _PyObject_Malloc(void *ctx, size_t size); +static void _PyObject_Free(void *ctx, void *p); +static void* _PyObject_Realloc(void *ctx, void *ptr, size_t size); +#endif + + +static void * +_PyMem_RawMalloc(void *ctx, size_t size) +{ + /* PyMem_Malloc(0) means malloc(1). Some systems would return NULL + for malloc(0), which would be treated as an error. Some platforms would + return a pointer with no memory behind it, which would break pymalloc. + To solve these problems, allocate an extra byte. */ + if (size == 0) + size = 1; + return malloc(size); +} + +static void * +_PyMem_RawRealloc(void *ctx, void *ptr, size_t size) +{ + if (size == 0) + size = 1; + return realloc(ptr, size); +} + +static void +_PyMem_RawFree(void *ctx, void *ptr) +{ + free(ptr); +} + +#define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawRealloc, _PyMem_RawFree +#ifdef WITH_PYMALLOC +#define PYOBJECT_FUNCS _PyObject_Malloc, _PyObject_Realloc, _PyObject_Free +#else +#define PYOBJECT_FUNCS PYRAW_FUNCS +#endif + +#ifdef PYMALLOC_DEBUG +typedef struct { + /* We tag each block with an API ID in order to tag API violations */ + char api_id; + PyMemAllocator alloc; +} debug_alloc_api_t; +static struct { + debug_alloc_api_t raw; + debug_alloc_api_t mem; + debug_alloc_api_t obj; +} _PyMem_Debug = { + {'r', {NULL, PYRAW_FUNCS}}, + {'m', {NULL, PYRAW_FUNCS}}, + {'o', {NULL, PYOBJECT_FUNCS}} + }; + +#define PYDEBUG_FUNCS _PyMem_DebugMallocCtx, _PyMem_DebugReallocCtx, _PyMem_DebugFreeCtx +#endif + +static PyMemAllocator _PyMem_Raw = { +#ifdef PYMALLOC_DEBUG + &_PyMem_Debug.raw, PYDEBUG_FUNCS +#else + NULL, PYRAW_FUNCS +#endif + }; + +static PyMemAllocator _PyMem = { +#ifdef PYMALLOC_DEBUG + &_PyMem_Debug.mem, PYDEBUG_FUNCS +#else + NULL, PYRAW_FUNCS +#endif + }; + +static PyMemAllocator _PyObject = { +#ifdef PYMALLOC_DEBUG + &_PyMem_Debug.obj, PYDEBUG_FUNCS +#else + NULL, PYOBJECT_FUNCS +#endif + }; + +#undef PYRAW_FUNCS +#undef PYOBJECT_FUNCS +#undef PYDEBUG_FUNCS + +void +PyMem_SetupDebugHooks(void) +{ +#ifdef PYMALLOC_DEBUG + PyMemAllocator alloc; + + alloc.malloc = _PyMem_DebugMallocCtx; + alloc.realloc = _PyMem_DebugReallocCtx; + alloc.free = _PyMem_DebugFreeCtx; + + if (_PyMem_Raw.malloc != _PyMem_DebugMallocCtx) { + alloc.ctx = &_PyMem_Debug.raw; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); + } + + if (_PyMem.malloc != _PyMem_DebugMallocCtx) { + alloc.ctx = &_PyMem_Debug.mem; + PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + } + + if (_PyObject.malloc != _PyMem_DebugMallocCtx) { + alloc.ctx = &_PyMem_Debug.obj; + PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + } +#endif +} + +void +PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator) +{ + switch(domain) + { + case PYMEM_DOMAIN_RAW: *allocator = _PyMem_Raw; break; + case PYMEM_DOMAIN_MEM: *allocator = _PyMem; break; + case PYMEM_DOMAIN_OBJ: *allocator = _PyObject; break; + default: + /* unknown domain */ + allocator->ctx = NULL; + allocator->malloc = NULL; + allocator->realloc = NULL; + allocator->free = NULL; + } +} + +void +PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocator *allocator) +{ + switch(domain) + { + case PYMEM_DOMAIN_RAW: _PyMem_Raw = *allocator; break; + case PYMEM_DOMAIN_MEM: _PyMem = *allocator; break; + case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break; + /* ignore unknown domain */ + } + +} + +void * +PyMem_RawMalloc(size_t size) +{ + /* + * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes. + * Most python internals blindly use a signed Py_ssize_t to track + * things without checking for overflows or negatives. + * As size_t is unsigned, checking for size < 0 is not required. + */ + if (size > (size_t)PY_SSIZE_T_MAX) + return NULL; + + return _PyMem_Raw.malloc(_PyMem_Raw.ctx, size); +} + +void* +PyMem_RawRealloc(void *ptr, size_t new_size) +{ + /* see PyMem_RawMalloc() */ + if (new_size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return _PyMem_Raw.realloc(_PyMem_Raw.ctx, ptr, new_size); +} + +void PyMem_RawFree(void *ptr) +{ + _PyMem_Raw.free(_PyMem_Raw.ctx, ptr); +} + +void * +PyMem_Malloc(size_t size) +{ + /* see PyMem_RawMalloc() */ + if (size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return _PyMem.malloc(_PyMem.ctx, size); +} + +void * +PyMem_Realloc(void *ptr, size_t new_size) +{ + /* see PyMem_RawMalloc() */ + if (new_size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return _PyMem.realloc(_PyMem.ctx, ptr, new_size); +} + +void +PyMem_Free(void *ptr) +{ + _PyMem.free(_PyMem.ctx, ptr); +} + +char * +_PyMem_RawStrdup(const char *str) +{ + size_t size; + char *copy; + + size = strlen(str) + 1; + copy = PyMem_RawMalloc(size); + if (copy == NULL) + return NULL; + memcpy(copy, str, size); + return copy; +} + +char * +_PyMem_Strdup(const char *str) +{ + size_t size; + char *copy; + + size = strlen(str) + 1; + copy = PyMem_Malloc(size); + if (copy == NULL) + return NULL; + memcpy(copy, str, size); + return copy; +} + +void * +PyObject_Malloc(size_t size) +{ + /* see PyMem_RawMalloc() */ + if (size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return _PyObject.malloc(_PyObject.ctx, size); +} + +void * +PyObject_Realloc(void *ptr, size_t new_size) +{ + /* see PyMem_RawMalloc() */ + if (new_size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return _PyObject.realloc(_PyObject.ctx, ptr, new_size); +} + +void +PyObject_Free(void *ptr) +{ + _PyObject.free(_PyObject.ctx, ptr); +} + + #ifdef WITH_PYMALLOC #ifdef HAVE_MMAP @@ -581,7 +856,7 @@ return NULL; /* overflow */ #endif nbytes = numarenas * sizeof(*arenas); - arenaobj = (struct arena_object *)realloc(arenas, nbytes); + arenaobj = (struct arena_object *)PyMem_Realloc(arenas, nbytes); if (arenaobj == NULL) return NULL; arenas = arenaobj; @@ -785,9 +1060,8 @@ * Unless the optimizer reorders everything, being too smart... */ -#undef PyObject_Malloc -void * -PyObject_Malloc(size_t nbytes) +static void * +_PyObject_Malloc(void *ctx, size_t nbytes) { block *bp; poolp pool; @@ -802,15 +1076,6 @@ #endif /* - * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes. - * Most python internals blindly use a signed Py_ssize_t to track - * things without checking for overflows or negatives. - * As size_t is unsigned, checking for nbytes < 0 is not required. - */ - if (nbytes > PY_SSIZE_T_MAX) - return NULL; - - /* * This implicitly redirects malloc(0). */ if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) { @@ -983,15 +1248,14 @@ */ if (nbytes == 0) nbytes = 1; - return (void *)malloc(nbytes); + return PyMem_Malloc(nbytes); } /* free */ -#undef PyObject_Free ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS -void -PyObject_Free(void *p) +static void +_PyObject_Free(void *ctx, void *p) { poolp pool; block *lastfree; @@ -1211,7 +1475,7 @@ redirect: #endif /* We didn't allocate this address. */ - free(p); + PyMem_Free(p); } /* realloc. If p is NULL, this acts like malloc(nbytes). Else if nbytes==0, @@ -1219,10 +1483,9 @@ * return a non-NULL result. */ -#undef PyObject_Realloc ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS -void * -PyObject_Realloc(void *p, size_t nbytes) +static void * +_PyObject_Realloc(void *ctx, void *p, size_t nbytes) { void *bp; poolp pool; @@ -1232,16 +1495,7 @@ #endif if (p == NULL) - return PyObject_Malloc(nbytes); - - /* - * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes. - * Most python internals blindly use a signed Py_ssize_t to track - * things without checking for overflows or negatives. - * As size_t is unsigned, checking for nbytes < 0 is not required. - */ - if (nbytes > PY_SSIZE_T_MAX) - return NULL; + return _PyObject_Malloc(ctx, nbytes); #ifdef WITH_VALGRIND /* Treat running_on_valgrind == -1 the same as 0 */ @@ -1269,10 +1523,10 @@ } size = nbytes; } - bp = PyObject_Malloc(nbytes); + bp = _PyObject_Malloc(ctx, nbytes); if (bp != NULL) { memcpy(bp, p, size); - PyObject_Free(p); + _PyObject_Free(ctx, p); } return bp; } @@ -1290,40 +1544,17 @@ * at p. Instead we punt: let C continue to manage this block. */ if (nbytes) - return realloc(p, nbytes); + return PyMem_Realloc(p, nbytes); /* C doesn't define the result of realloc(p, 0) (it may or may not * return NULL then), but Python's docs promise that nbytes==0 never * returns NULL. We don't pass 0 to realloc(), to avoid that endcase * to begin with. Even then, we can't be sure that realloc() won't * return NULL. */ - bp = realloc(p, 1); + bp = PyMem_Realloc(p, 1); return bp ? bp : p; } -#else /* ! WITH_PYMALLOC */ - -/*==========================================================================*/ -/* pymalloc not enabled: Redirect the entry points to malloc. These will - * only be used by extensions that are compiled with pymalloc enabled. */ - -void * -PyObject_Malloc(size_t n) -{ - return PyMem_MALLOC(n); -} - -void * -PyObject_Realloc(void *p, size_t n) -{ - return PyMem_REALLOC(p, n); -} - -void -PyObject_Free(void *p) -{ - PyMem_FREE(p); -} #endif /* WITH_PYMALLOC */ #ifdef PYMALLOC_DEBUG @@ -1343,10 +1574,6 @@ #define DEADBYTE 0xDB /* dead (newly freed) memory */ #define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */ -/* We tag each block with an API ID in order to tag API violations */ -#define _PYMALLOC_MEM_ID 'm' /* the PyMem_Malloc() API */ -#define _PYMALLOC_OBJ_ID 'o' /* The PyObject_Malloc() API */ - static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ /* serialno is always incremented via calling this routine. The point is @@ -1429,58 +1656,18 @@ p[2*S+n: 2*S+n+S] Copies of FORBIDDENBYTE. Used to catch over- writes and reads. p[2*S+n+S: 2*S+n+2*S] - A serial number, incremented by 1 on each call to _PyObject_DebugMalloc - and _PyObject_DebugRealloc. + A serial number, incremented by 1 on each call to _PyMem_DebugMalloc + and _PyMem_DebugRealloc. This is a big-endian size_t. If "bad memory" is detected later, the serial number gives an excellent way to set a breakpoint on the next run, to capture the instant at which this block was passed out. */ -/* debug replacements for the PyMem_* memory API */ -void * -_PyMem_DebugMalloc(size_t nbytes) -{ - return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes); -} -void * -_PyMem_DebugRealloc(void *p, size_t nbytes) -{ - return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes); -} -void -_PyMem_DebugFree(void *p) -{ - _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p); -} - -/* debug replacements for the PyObject_* memory API */ -void * -_PyObject_DebugMalloc(size_t nbytes) -{ - return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes); -} -void * -_PyObject_DebugRealloc(void *p, size_t nbytes) -{ - return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes); -} -void -_PyObject_DebugFree(void *p) -{ - _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p); -} -void -_PyObject_DebugCheckAddress(const void *p) -{ - _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p); -} - - -/* generic debug memory api, with an "id" to identify the API in use */ -void * -_PyObject_DebugMallocApi(char id, size_t nbytes) +static void * +_PyMem_DebugMallocCtx(void *ctx, size_t nbytes) { + debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; uchar *p; /* base address of malloc'ed block */ uchar *tail; /* p + 2*SST + nbytes == pointer to tail pad bytes */ size_t total; /* nbytes + 4*SST */ @@ -1491,14 +1678,14 @@ /* overflow: can't represent total as a size_t */ return NULL; - p = (uchar *)PyObject_Malloc(total); + p = (uchar *)api->alloc.malloc(api->alloc.ctx, total); if (p == NULL) return NULL; /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */ write_size_t(p, nbytes); - p[SST] = (uchar)id; - memset(p + SST + 1 , FORBIDDENBYTE, SST-1); + p[SST] = (uchar)api->api_id; + memset(p + SST + 1, FORBIDDENBYTE, SST-1); if (nbytes > 0) memset(p + 2*SST, CLEANBYTE, nbytes); @@ -1516,35 +1703,37 @@ Then fills the original bytes with DEADBYTE. Then calls the underlying free. */ -void -_PyObject_DebugFreeApi(char api, void *p) +static void +_PyMem_DebugFreeCtx(void *ctx, void *p) { + debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; uchar *q = (uchar *)p - 2*SST; /* address returned from malloc */ size_t nbytes; if (p == NULL) return; - _PyObject_DebugCheckAddressApi(api, p); + _PyMem_DebugCheckAddress(api->api_id, p); nbytes = read_size_t(q); nbytes += 4*SST; if (nbytes > 0) memset(q, DEADBYTE, nbytes); - PyObject_Free(q); + api->alloc.free(api->alloc.ctx, q); } -void * -_PyObject_DebugReallocApi(char api, void *p, size_t nbytes) +static void * +_PyMem_DebugReallocCtx(void *ctx, void *p, size_t nbytes) { - uchar *q = (uchar *)p; + debug_alloc_api_t *api = (debug_alloc_api_t *)ctx; + uchar *q = (uchar *)p, *oldq; uchar *tail; size_t total; /* nbytes + 4*SST */ size_t original_nbytes; int i; if (p == NULL) - return _PyObject_DebugMallocApi(api, nbytes); + return _PyMem_DebugMallocCtx(ctx, nbytes); - _PyObject_DebugCheckAddressApi(api, p); + _PyMem_DebugCheckAddress(api->api_id, p); bumpserialno(); original_nbytes = read_size_t(q - 2*SST); total = nbytes + 4*SST; @@ -1552,24 +1741,26 @@ /* overflow: can't represent total as a size_t */ return NULL; - if (nbytes < original_nbytes) { - /* shrinking: mark old extra memory dead */ - memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST); - } - /* Resize and add decorations. We may get a new pointer here, in which * case we didn't get the chance to mark the old memory with DEADBYTE, * but we live with that. */ - q = (uchar *)PyObject_Realloc(q - 2*SST, total); + oldq = q; + q = (uchar *)api->alloc.realloc(api->alloc.ctx, q - 2*SST, total); if (q == NULL) return NULL; + if (q == oldq && nbytes < original_nbytes) { + /* shrinking: mark old extra memory dead */ + memset(q + nbytes, DEADBYTE, original_nbytes - nbytes); + } + write_size_t(q, nbytes); - assert(q[SST] == (uchar)api); + assert(q[SST] == (uchar)api->api_id); for (i = 1; i < SST; ++i) assert(q[SST + i] == FORBIDDENBYTE); q += 2*SST; + tail = q + nbytes; memset(tail, FORBIDDENBYTE, SST); write_size_t(tail + SST, serialno); @@ -1588,8 +1779,8 @@ * and call Py_FatalError to kill the program. * The API id, is also checked. */ - void -_PyObject_DebugCheckAddressApi(char api, const void *p) +static void +_PyMem_DebugCheckAddress(char api, const void *p) { const uchar *q = (const uchar *)p; char msgbuf[64]; @@ -1935,3 +2126,44 @@ arenas[arenaindex_temp].address != 0; } #endif + + +#if defined(WITH_PYMALLOC) && defined(PYMALLOC_DEBUG) +/* Dummy functions only present to keep the same ABI with the vanilla Python + compiled in debug mode: they are not used in practice. See issue: + https://github.com/haypo/pytracemalloc/issues/1 */ + +void* _PyMem_DebugMalloc(size_t nbytes) +{ return PyMem_RawMalloc(nbytes); } + +void* _PyMem_DebugRealloc(void *p, size_t nbytes) +{ return PyMem_RawRealloc(p, nbytes); } + +void _PyObject_DebugFree(void *p) +{ return PyObject_Free(p); } + +void* _PyObject_DebugMalloc(size_t nbytes) +{ return PyObject_Malloc(nbytes); } + +void* _PyObject_DebugRealloc(void *p, size_t nbytes) +{ return PyObject_Realloc(p, nbytes); } + +void _PyMem_DebugFree(void *p) +{ PyMem_RawFree(p); } + +void _PyObject_DebugCheckAddress(const void *p) +{} + +void * _PyObject_DebugMallocApi(char api, size_t nbytes) +{ return PyObject_Malloc(nbytes); } + +void * _PyObject_DebugReallocApi(char api, void *p, size_t nbytes) +{ return PyObject_Realloc(p, nbytes); } + +void _PyObject_DebugFreeApi(char api, void *p) +{ return PyObject_Free(p); } + +void _PyObject_DebugCheckAddressApi(char api, const void *p) +{} +#endif + diff -urN Python-2.7.10.ORIG/Python/pythonrun.c Python-2.7.10/Python/pythonrun.c --- Python-2.7.10.ORIG/Python/pythonrun.c 2015-05-24 01:09:24.000000000 +0900 +++ Python-2.7.10/Python/pythonrun.c 2015-10-13 17:31:13.771317208 +0900 @@ -137,6 +137,41 @@ return flag; } +static void +inittracemalloc(void) +{ + PyObject *mod = NULL, *res = NULL; + char *p, *endptr; + long nframe; + + p = Py_GETENV("PYTHONTRACEMALLOC"); + if (p == NULL || *p == '\0') + return; + + endptr = p; + nframe = strtol(p, &endptr, 10); + if (*endptr != '\0' || nframe < 1 || nframe > 100000) + Py_FatalError("PYTHONTRACEMALLOC: invalid number of frames"); + + mod = PyImport_ImportModule("_tracemalloc"); + if (mod == NULL) + goto error; + + res = PyObject_CallMethod(mod, "start", "i", (int)nframe); + if (res == NULL) + goto error; + + goto done; + +error: + fprintf(stderr, "failed to start tracemalloc:\n"); + PyErr_Print(); + +done: + Py_XDECREF(mod); + Py_XDECREF(res); +} + void Py_InitializeEx(int install_sigs) { @@ -266,6 +301,8 @@ _PyGILState_Init(interp, tstate); #endif /* WITH_THREAD */ + inittracemalloc(); + if (!Py_NoSiteFlag) initsite(); /* Module site */