[Python-checkins] cpython: On memory error, dump the memory block traceback
victor.stinner
python-checkins at python.org
Tue Mar 15 17:42:23 EDT 2016
https://hg.python.org/cpython/rev/cef6a32d805f
changeset: 100551:cef6a32d805f
user: Victor Stinner <victor.stinner at gmail.com>
date: Tue Mar 15 22:22:13 2016 +0100
summary:
On memory error, dump the memory block traceback
Issue #26564: _PyObject_DebugDumpAddress() now dumps the traceback where a
memory block was allocated on memory block. Use the tracemalloc module to get
the traceback.
files:
Doc/c-api/memory.rst | 7 ++
Doc/whatsnew/3.6.rst | 43 ++++++++++++++++-
Misc/NEWS | 4 +
Modules/_tracemalloc.c | 67 ++++++++++++++++++++++----
Modules/hashtable.c | 6 +-
Objects/bytearrayobject.c | 1 +
Objects/obmalloc.c | 9 +++
Parser/pgenmain.c | 8 +-
8 files changed, 126 insertions(+), 19 deletions(-)
diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst
--- a/Doc/c-api/memory.rst
+++ b/Doc/c-api/memory.rst
@@ -349,12 +349,19 @@
allocator functions of the :c:data:`PYMEM_DOMAIN_OBJ` domain (ex:
:c:func:`PyObject_Malloc`) are called
+ On error, the debug hooks use the :mod:`tracemalloc` module to get the
+ traceback where a memory block was allocated. The traceback is only
+ displayed if :mod:`tracemalloc` is tracing Python memory allocations and the
+ memory block was traced.
+
These hooks are installed by default if Python is compiled in debug
mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install
debug hooks on a Python compiled in release mode.
.. versionchanged:: 3.6
This function now also works on Python compiled in release mode.
+ On error, the debug hooks now use :mod:`tracemalloc` to get the traceback
+ where a memory block was allocated.
.. _pymalloc:
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -129,7 +129,48 @@
It helps to use external memory debuggers like Valgrind on a Python compiled in
release mode.
-(Contributed by Victor Stinner in :issue:`26516`.)
+On error, the debug hooks on Python memory allocators now use the
+:mod:`tracemalloc` module to get the traceback where a memory block was
+allocated.
+
+Example of fatal error on buffer overflow using
+``python3.6 -X tracemalloc=5`` (store 5 frames in traces)::
+
+ Debug memory block at address p=0x7fbcd41666f8: API 'o'
+ 4 bytes originally requested
+ The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.
+ The 8 pad bytes at tail=0x7fbcd41666fc are not all FORBIDDENBYTE (0xfb):
+ at tail+0: 0x02 *** OUCH
+ at tail+1: 0xfb
+ at tail+2: 0xfb
+ at tail+3: 0xfb
+ at tail+4: 0xfb
+ at tail+5: 0xfb
+ at tail+6: 0xfb
+ at tail+7: 0xfb
+ The block was made by call #1233329 to debug malloc/realloc.
+ Data at p: 1a 2b 30 00
+
+ Memory block allocated at (most recent call first):
+ File "test/test_bytes.py", line 323
+ File "unittest/case.py", line 600
+ File "unittest/case.py", line 648
+ File "unittest/suite.py", line 122
+ File "unittest/suite.py", line 84
+
+ Fatal Python error: bad trailing pad byte
+
+ Current thread 0x00007fbcdbd32700 (most recent call first):
+ File "test/test_bytes.py", line 323 in test_hex
+ File "unittest/case.py", line 600 in run
+ File "unittest/case.py", line 648 in __call__
+ File "unittest/suite.py", line 122 in run
+ File "unittest/suite.py", line 84 in __call__
+ File "unittest/suite.py", line 122 in run
+ File "unittest/suite.py", line 84 in __call__
+ ...
+
+(Contributed by Victor Stinner in :issue:`26516` and :issue:`26564`.)
Other Language Changes
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@
Core and Builtins
-----------------
+- Issue #26564: On error, the debug hooks on Python memory allocators now use
+ the :mod:`tracemalloc` module to get the traceback where a memory block was
+ allocated.
+
- Issue #26558: The debug hooks on Python memory allocator
:c:func:`PyObject_Malloc` now detect when functions are called without
holding the GIL.
diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c
--- a/Modules/_tracemalloc.c
+++ b/Modules/_tracemalloc.c
@@ -1161,6 +1161,25 @@
return get_traces.list;
}
+static traceback_t*
+tracemalloc_get_traceback(const void *ptr)
+{
+ trace_t trace;
+ int found;
+
+ if (!tracemalloc_config.tracing)
+ return NULL;
+
+ TABLES_LOCK();
+ found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace);
+ TABLES_UNLOCK();
+
+ if (!found)
+ return NULL;
+
+ return trace.traceback;
+}
+
PyDoc_STRVAR(tracemalloc_get_object_traceback_doc,
"_get_object_traceback(obj)\n"
"\n"
@@ -1175,11 +1194,7 @@
{
PyTypeObject *type;
void *ptr;
- trace_t trace;
- int found;
-
- if (!tracemalloc_config.tracing)
- Py_RETURN_NONE;
+ traceback_t *traceback;
type = Py_TYPE(obj);
if (PyType_IS_GC(type))
@@ -1187,16 +1202,46 @@
else
ptr = (void *)obj;
- TABLES_LOCK();
- found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace);
- TABLES_UNLOCK();
-
- if (!found)
+ traceback = tracemalloc_get_traceback(ptr);
+ if (traceback == NULL)
Py_RETURN_NONE;
- return traceback_to_pyobject(trace.traceback, NULL);
+ return traceback_to_pyobject(traceback, NULL);
}
+#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str))
+
+static void
+_PyMem_DumpFrame(int fd, frame_t * frame)
+{
+ PUTS(fd, " File \"");
+ _Py_DumpASCII(fd, frame->filename);
+ PUTS(fd, "\", line ");
+ _Py_DumpDecimal(fd, frame->lineno);
+ PUTS(fd, "\n");
+}
+
+/* Dump the traceback where a memory block was allocated into file descriptor
+ fd. The function may block on TABLES_LOCK() but it is unlikely. */
+void
+_PyMem_DumpTraceback(int fd, const void *ptr)
+{
+ traceback_t *traceback;
+ int i;
+
+ traceback = tracemalloc_get_traceback(ptr);
+ if (traceback == NULL)
+ return;
+
+ PUTS(fd, "Memory block allocated at (most recent call first):\n");
+ for (i=0; i < traceback->nframe; i++) {
+ _PyMem_DumpFrame(fd, &traceback->frames[i]);
+ }
+ PUTS(fd, "\n");
+}
+
+#undef PUTS
+
PyDoc_STRVAR(tracemalloc_start_doc,
"start(nframe: int=1)\n"
"\n"
diff --git a/Modules/hashtable.c b/Modules/hashtable.c
--- a/Modules/hashtable.c
+++ b/Modules/hashtable.c
@@ -486,9 +486,9 @@
void *data, *new_data;
dst = _Py_hashtable_new_full(src->data_size, src->num_buckets,
- src->hash_func, src->compare_func,
- src->copy_data_func, src->free_data_func,
- src->get_data_size_func, &src->alloc);
+ src->hash_func, src->compare_func,
+ src->copy_data_func, src->free_data_func,
+ src->get_data_size_func, &src->alloc);
if (dst == NULL)
return NULL;
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
--- a/Objects/bytearrayobject.c
+++ b/Objects/bytearrayobject.c
@@ -2820,6 +2820,7 @@
{
char* argbuf = PyByteArray_AS_STRING(self);
Py_ssize_t arglen = PyByteArray_GET_SIZE(self);
+ PyByteArray_AS_STRING(self)[arglen+1] = 2;
return _Py_strhex(argbuf, arglen);
}
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -1,5 +1,10 @@
#include "Python.h"
+
+/* Defined in tracemalloc.c */
+extern void _PyMem_DumpTraceback(int fd, const void *ptr);
+
+
/* Python's malloc wrappers (see pymem.h) */
/*
@@ -2202,6 +2207,10 @@
}
fputc('\n', stderr);
}
+ fputc('\n', stderr);
+
+ fflush(stderr);
+ _PyMem_DumpTraceback(fileno(stderr), p);
}
diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c
--- a/Parser/pgenmain.c
+++ b/Parser/pgenmain.c
@@ -38,11 +38,11 @@
}
#ifdef WITH_THREAD
-/* Needed by obmalloc.c */
+/* Functions needed by obmalloc.c */
int PyGILState_Check(void)
-{
- return 1;
-}
+{ return 1; }
+void _PyMem_DumpTraceback(int fd, const void *ptr)
+{}
#endif
int
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list