[Python-checkins] r75105 - in python/branches/py3k: Include/objimpl.h Include/pymem.h Objects/obmalloc.c Parser/parsetok.c
kristjan.jonsson
python-checkins at python.org
Mon Sep 28 15:45:02 CEST 2009
Author: kristjan.jonsson
Date: Mon Sep 28 15:45:02 2009
New Revision: 75105
Log:
http://bugs.python.org/issue6836
Merging revisions 75103,75104 from trunk to py3k
Modified:
python/branches/py3k/Include/objimpl.h
python/branches/py3k/Include/pymem.h
python/branches/py3k/Objects/obmalloc.c
python/branches/py3k/Parser/parsetok.c
Modified: python/branches/py3k/Include/objimpl.h
==============================================================================
--- python/branches/py3k/Include/objimpl.h (original)
+++ python/branches/py3k/Include/objimpl.h Mon Sep 28 15:45:02 2009
@@ -108,6 +108,13 @@
PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
PyAPI_FUNC(void) _PyObject_DebugMallocStats(void);
+PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
+PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
+PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
+PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
+PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
#define PyObject_MALLOC _PyObject_DebugMalloc
#define PyObject_Malloc _PyObject_DebugMalloc
#define PyObject_REALLOC _PyObject_DebugRealloc
Modified: python/branches/py3k/Include/pymem.h
==============================================================================
--- python/branches/py3k/Include/pymem.h (original)
+++ python/branches/py3k/Include/pymem.h Mon Sep 28 15:45:02 2009
@@ -59,9 +59,9 @@
/* Macros. */
#ifdef PYMALLOC_DEBUG
/* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC PyObject_MALLOC
-#define PyMem_REALLOC PyObject_REALLOC
-#define PyMem_FREE PyObject_FREE
+#define PyMem_MALLOC _PyMem_DebugMalloc
+#define PyMem_REALLOC _PyMem_DebugRealloc
+#define PyMem_FREE _PyMem_DebugFree
#else /* ! PYMALLOC_DEBUG */
Modified: python/branches/py3k/Objects/obmalloc.c
==============================================================================
--- python/branches/py3k/Objects/obmalloc.c (original)
+++ python/branches/py3k/Objects/obmalloc.c Mon Sep 28 15:45:02 2009
@@ -1241,6 +1241,10 @@
#define DEADBYTE 0xDB /* dead (newly freed) memory */
#define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */
+/* We tag each block with an API ID in order to tag API violations */
+#define _PYMALLOC_MEM_ID 'm' /* the PyMem_Malloc() API */
+#define _PYMALLOC_OBJ_ID 'o' /* The PyObject_Malloc() API */
+
static size_t serialno = 0; /* incremented on each debug {m,re}alloc */
/* serialno is always incremented via calling this routine. The point is
@@ -1331,9 +1335,50 @@
instant at which this block was passed out.
*/
+/* debug replacements for the PyMem_* memory API */
+void *
+_PyMem_DebugMalloc(size_t nbytes)
+{
+ return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes);
+}
+void *
+_PyMem_DebugRealloc(void *p, size_t nbytes)
+{
+ return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes);
+}
+void
+_PyMem_DebugFree(void *p)
+{
+ _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p);
+}
+
+/* debug replacements for the PyObject_* memory API */
void *
_PyObject_DebugMalloc(size_t nbytes)
{
+ return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes);
+}
+void *
+_PyObject_DebugRealloc(void *p, size_t nbytes)
+{
+ return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes);
+}
+void
+_PyObject_DebugFree(void *p)
+{
+ _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p);
+}
+void
+_PyObject_DebugCheckAddress(void *p)
+{
+ _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p);
+}
+
+
+/* generic debug memory api, with an "id" to identify the API in use */
+void *
+_PyObject_DebugMallocApi(char id, size_t nbytes)
+{
uchar *p; /* base address of malloc'ed block */
uchar *tail; /* p + 2*SST + nbytes == pointer to tail pad bytes */
size_t total; /* nbytes + 4*SST */
@@ -1348,12 +1393,15 @@
if (p == NULL)
return NULL;
+ /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
write_size_t(p, nbytes);
- memset(p + SST, FORBIDDENBYTE, SST);
+ p[SST] = (uchar)id;
+ memset(p + SST + 1 , FORBIDDENBYTE, SST-1);
if (nbytes > 0)
memset(p + 2*SST, CLEANBYTE, nbytes);
+ /* at tail, write pad (SST bytes) and serialno (SST bytes) */
tail = p + 2*SST + nbytes;
memset(tail, FORBIDDENBYTE, SST);
write_size_t(tail + SST, serialno);
@@ -1362,27 +1410,28 @@
}
/* The debug free first checks the 2*SST bytes on each end for sanity (in
- particular, that the FORBIDDENBYTEs are still intact).
+ particular, that the FORBIDDENBYTEs with the api ID are still intact).
Then fills the original bytes with DEADBYTE.
Then calls the underlying free.
*/
void
-_PyObject_DebugFree(void *p)
+_PyObject_DebugFreeApi(char api, void *p)
{
uchar *q = (uchar *)p - 2*SST; /* address returned from malloc */
size_t nbytes;
if (p == NULL)
return;
- _PyObject_DebugCheckAddress(p);
+ _PyObject_DebugCheckAddressApi(api, p);
nbytes = read_size_t(q);
+ nbytes += 4*SST;
if (nbytes > 0)
memset(q, DEADBYTE, nbytes);
PyObject_Free(q);
}
void *
-_PyObject_DebugRealloc(void *p, size_t nbytes)
+_PyObject_DebugReallocApi(char api, void *p, size_t nbytes)
{
uchar *q = (uchar *)p;
uchar *tail;
@@ -1391,9 +1440,9 @@
int i;
if (p == NULL)
- return _PyObject_DebugMalloc(nbytes);
+ return _PyObject_DebugMallocApi(api, nbytes);
- _PyObject_DebugCheckAddress(p);
+ _PyObject_DebugCheckAddressApi(api, p);
bumpserialno();
original_nbytes = read_size_t(q - 2*SST);
total = nbytes + 4*SST;
@@ -1403,16 +1452,20 @@
if (nbytes < original_nbytes) {
/* shrinking: mark old extra memory dead */
- memset(q + nbytes, DEADBYTE, original_nbytes - nbytes);
+ memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST);
}
- /* Resize and add decorations. */
+ /* Resize and add decorations. We may get a new pointer here, in which
+ * case we didn't get the chance to mark the old memory with DEADBYTE,
+ * but we live with that.
+ */
q = (uchar *)PyObject_Realloc(q - 2*SST, total);
if (q == NULL)
return NULL;
write_size_t(q, nbytes);
- for (i = 0; i < SST; ++i)
+ assert(q[SST] == (uchar)api);
+ for (i = 1; i < SST; ++i)
assert(q[SST + i] == FORBIDDENBYTE);
q += 2*SST;
tail = q + nbytes;
@@ -1431,26 +1484,38 @@
/* Check the forbidden bytes on both ends of the memory allocated for p.
* If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress,
* and call Py_FatalError to kill the program.
+ * The API id, is also checked.
*/
void
-_PyObject_DebugCheckAddress(const void *p)
+_PyObject_DebugCheckAddressApi(char api, const void *p)
{
const uchar *q = (const uchar *)p;
+ char msgbuf[64];
char *msg;
size_t nbytes;
const uchar *tail;
int i;
+ char id;
if (p == NULL) {
msg = "didn't expect a NULL pointer";
goto error;
}
+ /* Check the API id */
+ id = (char)q[-SST];
+ if (id != api) {
+ msg = msgbuf;
+ snprintf(msg, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api);
+ msgbuf[sizeof(msgbuf)-1] = 0;
+ goto error;
+ }
+
/* Check the stuff at the start of p first: if there's underwrite
* corruption, the number-of-bytes field may be nuts, and checking
* the tail could lead to a segfault then.
*/
- for (i = SST; i >= 1; --i) {
+ for (i = SST-1; i >= 1; --i) {
if (*(q-i) != FORBIDDENBYTE) {
msg = "bad leading pad byte";
goto error;
@@ -1482,19 +1547,24 @@
size_t nbytes, serial;
int i;
int ok;
+ char id;
- fprintf(stderr, "Debug memory block at address p=%p:\n", p);
- if (p == NULL)
+ fprintf(stderr, "Debug memory block at address p=%p:", p);
+ if (p == NULL) {
+ fprintf(stderr, "\n");
return;
+ }
+ id = (char)q[-SST];
+ fprintf(stderr, " API '%c'\n", id);
nbytes = read_size_t(q - 2*SST);
fprintf(stderr, " %" PY_FORMAT_SIZE_T "u bytes originally "
"requested\n", nbytes);
/* In case this is nuts, check the leading pad bytes first. */
- fprintf(stderr, " The %d pad bytes at p-%d are ", SST, SST);
+ fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1);
ok = 1;
- for (i = 1; i <= SST; ++i) {
+ for (i = 1; i <= SST-1; ++i) {
if (*(q-i) != FORBIDDENBYTE) {
ok = 0;
break;
@@ -1505,7 +1575,7 @@
else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
FORBIDDENBYTE);
- for (i = SST; i >= 1; --i) {
+ for (i = SST-1; i >= 1; --i) {
const uchar byte = *(q-i);
fprintf(stderr, " at p-%d: 0x%02x", i, byte);
if (byte != FORBIDDENBYTE)
Modified: python/branches/py3k/Parser/parsetok.c
==============================================================================
--- python/branches/py3k/Parser/parsetok.c (original)
+++ python/branches/py3k/Parser/parsetok.c Mon Sep 28 15:45:02 2009
@@ -240,16 +240,24 @@
}
}
} else if (tok->encoding != NULL) {
+ /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was
+ * allocated using PyMem_
+ */
node* r = PyNode_New(encoding_decl);
- if (!r) {
+ if (r)
+ r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1);
+ if (!r || !r->n_str) {
err_ret->error = E_NOMEM;
+ if (r)
+ PyObject_FREE(r);
n = NULL;
goto done;
}
- r->n_str = tok->encoding;
+ strcpy(r->n_str, tok->encoding);
+ PyMem_FREE(tok->encoding);
+ tok->encoding = NULL;
r->n_nchildren = 1;
r->n_child = n;
- tok->encoding = NULL;
n = r;
}
More information about the Python-checkins
mailing list