http://bugs.python.org/issue6836

Merging revisions 75103,75104 from trunk to py3k

Include/objimpl.h

@@ -108,6 +108,13 @@ PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
 PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
 PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
 PyAPI_FUNC(void) _PyObject_DebugMallocStats(void);
+PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
+PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
+PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
+PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
+PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
 #define PyObject_MALLOC		_PyObject_DebugMalloc
 #define PyObject_Malloc		_PyObject_DebugMalloc
 #define PyObject_REALLOC	_PyObject_DebugRealloc

Include/pymem.h

@@ -59,9 +59,9 @@ PyAPI_FUNC(void) PyMem_Free(void *);
 /* Macros. */
 #ifdef PYMALLOC_DEBUG
 /* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC		PyObject_MALLOC
-#define PyMem_REALLOC		PyObject_REALLOC
-#define PyMem_FREE		PyObject_FREE
+#define PyMem_MALLOC		_PyMem_DebugMalloc
+#define PyMem_REALLOC		_PyMem_DebugRealloc
+#define PyMem_FREE		_PyMem_DebugFree
 
 #else	/* ! PYMALLOC_DEBUG */
 

Objects/obmalloc.c

@@ -1241,6 +1241,10 @@ PyObject_Free(void *p)
 #define DEADBYTE       0xDB    /* dead (newly freed) memory */
 #define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */
 
+/* We tag each block with an API ID in order to tag API violations */
+#define _PYMALLOC_MEM_ID 'm'   /* the PyMem_Malloc() API */
+#define _PYMALLOC_OBJ_ID 'o'   /* The PyObject_Malloc() API */
+
 static size_t serialno = 0;	/* incremented on each debug {m,re}alloc */
 
 /* serialno is always incremented via calling this routine.  The point is
@@ -1331,8 +1335,49 @@ p[2*S+n+S: 2*S+n+2*S]
     instant at which this block was passed out.
 */
 
+/* debug replacements for the PyMem_* memory API */
+void *
+_PyMem_DebugMalloc(size_t nbytes)
+{
+	return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes);
+}
+void *
+_PyMem_DebugRealloc(void *p, size_t nbytes)
+{
+	return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes);
+}
+void
+_PyMem_DebugFree(void *p)
+{
+	_PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p);
+}
+
+/* debug replacements for the PyObject_* memory API */
 void *
 _PyObject_DebugMalloc(size_t nbytes)
+{
+	return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes);
+}
+void *
+_PyObject_DebugRealloc(void *p, size_t nbytes)
+{
+	return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes);
+}
+void
+_PyObject_DebugFree(void *p)
+{
+	_PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p);
+}
+void
+_PyObject_DebugCheckAddress(void *p)
+{
+	_PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p);
+}
+
+
+/* generic debug memory api, with an "id" to identify the API in use */
+void *
+_PyObject_DebugMallocApi(char id, size_t nbytes)
 {
 	uchar *p;	/* base address of malloc'ed block */
 	uchar *tail;	/* p + 2*SST + nbytes == pointer to tail pad bytes */
@@ -1348,12 +1393,15 @@ _PyObject_DebugMalloc(size_t nbytes)
 	if (p == NULL)
 		return NULL;
 
+	/* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
 	write_size_t(p, nbytes);
-	memset(p + SST, FORBIDDENBYTE, SST);
+	p[SST] = (uchar)id;
+	memset(p + SST + 1 , FORBIDDENBYTE, SST-1);
 
 	if (nbytes > 0)
 		memset(p + 2*SST, CLEANBYTE, nbytes);
 
+	/* at tail, write pad (SST bytes) and serialno (SST bytes) */
 	tail = p + 2*SST + nbytes;
 	memset(tail, FORBIDDENBYTE, SST);
 	write_size_t(tail + SST, serialno);
@@ -1362,27 +1410,28 @@ _PyObject_DebugMalloc(size_t nbytes)
 }
 
 /* The debug free first checks the 2*SST bytes on each end for sanity (in
-   particular, that the FORBIDDENBYTEs are still intact).
+   particular, that the FORBIDDENBYTEs with the api ID are still intact).
    Then fills the original bytes with DEADBYTE.
    Then calls the underlying free.
 */
 void
-_PyObject_DebugFree(void *p)
+_PyObject_DebugFreeApi(char api, void *p)
 {
 	uchar *q = (uchar *)p - 2*SST;  /* address returned from malloc */
 	size_t nbytes;
 
 	if (p == NULL)
 		return;
-	_PyObject_DebugCheckAddress(p);
+	_PyObject_DebugCheckAddressApi(api, p);
 	nbytes = read_size_t(q);
+	nbytes += 4*SST;
 	if (nbytes > 0)
 		memset(q, DEADBYTE, nbytes);
 	PyObject_Free(q);
 }
 
 void *
-_PyObject_DebugRealloc(void *p, size_t nbytes)
+_PyObject_DebugReallocApi(char api, void *p, size_t nbytes)
 {
 	uchar *q = (uchar *)p;
 	uchar *tail;
@@ -1391,9 +1440,9 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
 	int i;
 
 	if (p == NULL)
-		return _PyObject_DebugMalloc(nbytes);
+		return _PyObject_DebugMallocApi(api, nbytes);
 
-	_PyObject_DebugCheckAddress(p);
+	_PyObject_DebugCheckAddressApi(api, p);
 	bumpserialno();
 	original_nbytes = read_size_t(q - 2*SST);
 	total = nbytes + 4*SST;
@@ -1403,16 +1452,20 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
 
 	if (nbytes < original_nbytes) {
 		/* shrinking:  mark old extra memory dead */
-		memset(q + nbytes, DEADBYTE, original_nbytes - nbytes);
+		memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST);
 	}
 
-	/* Resize and add decorations. */
+	/* Resize and add decorations. We may get a new pointer here, in which
+	 * case we didn't get the chance to mark the old memory with DEADBYTE,
+	 * but we live with that.
+	 */
 	q = (uchar *)PyObject_Realloc(q - 2*SST, total);
 	if (q == NULL)
 		return NULL;
 
 	write_size_t(q, nbytes);
-	for (i = 0; i < SST; ++i)
+	assert(q[SST] == (uchar)api);
+	for (i = 1; i < SST; ++i)
 		assert(q[SST + i] == FORBIDDENBYTE);
 	q += 2*SST;
 	tail = q + nbytes;
@@ -1431,26 +1484,38 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
 /* Check the forbidden bytes on both ends of the memory allocated for p.
  * If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress,
  * and call Py_FatalError to kill the program.
+ * The API id, is also checked.
  */
  void
-_PyObject_DebugCheckAddress(const void *p)
+_PyObject_DebugCheckAddressApi(char api, const void *p)
 {
 	const uchar *q = (const uchar *)p;
+	char msgbuf[64];
 	char *msg;
 	size_t nbytes;
 	const uchar *tail;
 	int i;
+	char id;
 
 	if (p == NULL) {
 		msg = "didn't expect a NULL pointer";
 		goto error;
 	}
 
+	/* Check the API id */
+	id = (char)q[-SST];
+	if (id != api) {
+		msg = msgbuf;
+		snprintf(msg, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api);
+		msgbuf[sizeof(msgbuf)-1] = 0;
+		goto error;
+	}
+
 	/* Check the stuff at the start of p first:  if there's underwrite
 	 * corruption, the number-of-bytes field may be nuts, and checking
 	 * the tail could lead to a segfault then.
 	 */
-	for (i = SST; i >= 1; --i) {
+	for (i = SST-1; i >= 1; --i) {
 		if (*(q-i) != FORBIDDENBYTE) {
 			msg = "bad leading pad byte";
 			goto error;
@@ -1482,19 +1547,24 @@ _PyObject_DebugDumpAddress(const void *p)
 	size_t nbytes, serial;
 	int i;
 	int ok;
+	char id;
 
-	fprintf(stderr, "Debug memory block at address p=%p:\n", p);
-	if (p == NULL)
+	fprintf(stderr, "Debug memory block at address p=%p:", p);
+	if (p == NULL) {
+		fprintf(stderr, "\n");
 		return;
+	}
+	id = (char)q[-SST];
+	fprintf(stderr, " API '%c'\n", id);
 
 	nbytes = read_size_t(q - 2*SST);
 	fprintf(stderr, "    %" PY_FORMAT_SIZE_T "u bytes originally "
 	                "requested\n", nbytes);
 
 	/* In case this is nuts, check the leading pad bytes first. */
-	fprintf(stderr, "    The %d pad bytes at p-%d are ", SST, SST);
+	fprintf(stderr, "    The %d pad bytes at p-%d are ", SST-1, SST-1);
 	ok = 1;
-	for (i = 1; i <= SST; ++i) {
+	for (i = 1; i <= SST-1; ++i) {
 		if (*(q-i) != FORBIDDENBYTE) {
 			ok = 0;
 			break;
@@ -1505,7 +1575,7 @@ _PyObject_DebugDumpAddress(const void *p)
 	else {
 		fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
 			FORBIDDENBYTE);
-		for (i = SST; i >= 1; --i) {
+		for (i = SST-1; i >= 1; --i) {
 			const uchar byte = *(q-i);
 			fprintf(stderr, "        at p-%d: 0x%02x", i, byte);
 			if (byte != FORBIDDENBYTE)

Parser/parsetok.c

@@ -240,16 +240,24 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
 			}
 		}
 	} else if (tok->encoding != NULL) {
+		/* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
+		 * allocated using PyMem_
+		 */
 		node* r = PyNode_New(encoding_decl);
-		if (!r) {
+		if (r)
+			r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
+		if (!r || !r->n_str) {
 			err_ret->error = E_NOMEM;
+			if (r)
+				PyObject_FREE(r);
 			n = NULL;
 			goto done;
 		}
-		r->n_str = tok->encoding;
+		strcpy(r->n_str, tok->encoding);
+		PyMem_FREE(tok->encoding);
+		tok->encoding = NULL;
 		r->n_nchildren = 1;
 		r->n_child = n;
-		tok->encoding = NULL;
 		n = r;
 	}