[Python-checkins] cpython: Issue #14785: Add sys._debugmallocstats() to help debug low-level memory
david.malcolm
python-checkins at python.org
Fri Jun 22 20:56:56 CEST 2012
http://hg.python.org/cpython/rev/d63a80abfbec
changeset: 77577:d63a80abfbec
user: David Malcolm <dmalcolm at redhat.com>
date: Fri Jun 22 14:55:41 2012 -0400
summary:
Issue #14785: Add sys._debugmallocstats() to help debug low-level memory allocation issues
files:
Doc/library/sys.rst | 16 +++++
Include/dictobject.h | 1 +
Include/floatobject.h | 2 +
Include/frameobject.h | 2 +
Include/listobject.h | 1 +
Include/methodobject.h | 5 +
Include/object.h | 8 ++
Include/objimpl.h | 4 +-
Include/setobject.h | 1 +
Include/tupleobject.h | 3 +
Lib/test/test_sys.py | 6 ++
Misc/NEWS | 3 +
Objects/classobject.c | 9 +++
Objects/dictobject.c | 9 +++
Objects/floatobject.c | 10 +++
Objects/frameobject.c | 10 +++
Objects/listobject.c | 9 +++
Objects/methodobject.c | 9 +++
Objects/object.c | 12 ++++
Objects/obmalloc.c | 81 +++++++++++++++++------------
Objects/setobject.c | 10 +++
Objects/tupleobject.c | 16 +++++
Python/pythonrun.c | 2 +-
Python/sysmodule.c | 23 ++++++++
24 files changed, 217 insertions(+), 35 deletions(-)
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst
--- a/Doc/library/sys.rst
+++ b/Doc/library/sys.rst
@@ -106,6 +106,22 @@
This function should be used for internal and specialized purposes only.
+.. function:: _debugmallocstats()
+
+ Print low-level information to stderr about the state of CPython's memory
+ allocator.
+
+ If Python is configured --with-pydebug, it also performs some expensive
+ internal consistency checks.
+
+ .. versionadded:: 3.3
+
+ .. impl-detail::
+
+ This function is specific to CPython. The exact output format is not
+ defined here, and may change.
+
+
.. data:: dllhandle
Integer specifying the handle of the Python DLL. Availability: Windows.
diff --git a/Include/dictobject.h b/Include/dictobject.h
--- a/Include/dictobject.h
+++ b/Include/dictobject.h
@@ -111,6 +111,7 @@
#ifndef Py_LIMITED_API
int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value);
PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *);
+PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out);
#endif
#ifdef __cplusplus
diff --git a/Include/floatobject.h b/Include/floatobject.h
--- a/Include/floatobject.h
+++ b/Include/floatobject.h
@@ -110,6 +110,8 @@
/* free list api */
PyAPI_FUNC(int) PyFloat_ClearFreeList(void);
+PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out);
+
/* Format the object based on the format_spec, as defined in PEP 3101
(Advanced String Formatting). */
PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter(
diff --git a/Include/frameobject.h b/Include/frameobject.h
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -79,6 +79,8 @@
PyAPI_FUNC(int) PyFrame_ClearFreeList(void);
+PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out);
+
/* Return the line of code the frame is currently executing. */
PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *);
diff --git a/Include/listobject.h b/Include/listobject.h
--- a/Include/listobject.h
+++ b/Include/listobject.h
@@ -64,6 +64,7 @@
PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *);
PyAPI_FUNC(int) PyList_ClearFreeList(void);
+PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out);
#endif
/* Macro, trading safety for speed */
diff --git a/Include/methodobject.h b/Include/methodobject.h
--- a/Include/methodobject.h
+++ b/Include/methodobject.h
@@ -82,6 +82,11 @@
PyAPI_FUNC(int) PyCFunction_ClearFreeList(void);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _PyCFunction_DebugMallocStats(FILE *out);
+PyAPI_FUNC(void) _PyMethod_DebugMallocStats(FILE *out);
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/object.h b/Include/object.h
--- a/Include/object.h
+++ b/Include/object.h
@@ -977,6 +977,14 @@
else \
_PyTrash_deposit_object((PyObject*)op);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void)
+_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks,
+ size_t sizeof_block);
+PyAPI_FUNC(void)
+_PyObject_DebugTypeStats(FILE *out);
+#endif /* ifndef Py_LIMITED_API */
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/objimpl.h b/Include/objimpl.h
--- a/Include/objimpl.h
+++ b/Include/objimpl.h
@@ -101,13 +101,15 @@
/* Macros */
#ifdef WITH_PYMALLOC
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out);
+#endif /* #ifndef Py_LIMITED_API */
#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */
PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes);
PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes);
PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
-PyAPI_FUNC(void) _PyObject_DebugMallocStats(void);
PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
diff --git a/Include/setobject.h b/Include/setobject.h
--- a/Include/setobject.h
+++ b/Include/setobject.h
@@ -101,6 +101,7 @@
PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable);
PyAPI_FUNC(int) PySet_ClearFreeList(void);
+PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out);
#endif
#ifdef __cplusplus
diff --git a/Include/tupleobject.h b/Include/tupleobject.h
--- a/Include/tupleobject.h
+++ b/Include/tupleobject.h
@@ -63,6 +63,9 @@
#endif
PyAPI_FUNC(int) PyTuple_ClearFreeList(void);
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out);
+#endif /* Py_LIMITED_API */
#ifdef __cplusplus
}
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
--- a/Lib/test/test_sys.py
+++ b/Lib/test/test_sys.py
@@ -603,6 +603,12 @@
self.assertEqual(sys.implementation.name,
sys.implementation.name.lower())
+ def test_debugmallocstats(self):
+ # Test sys._debugmallocstats()
+ from test.script_helper import assert_python_ok
+ args = ['-c', 'import sys; sys._debugmallocstats()']
+ ret, out, err = assert_python_ok(*args)
+ self.assertIn(b"free PyDictObjects", err)
class SizeofTest(unittest.TestCase):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -147,6 +147,9 @@
- Issue #14963: Convert contextlib.ExitStack.__exit__ to use an iterative
algorithm (Patch by Alon Horev)
+- Issue #14785: Add sys._debugmallocstats() to help debug low-level memory
+ allocation issues
+
C-API
-----
diff --git a/Objects/classobject.c b/Objects/classobject.c
--- a/Objects/classobject.c
+++ b/Objects/classobject.c
@@ -400,6 +400,15 @@
(void)PyMethod_ClearFreeList();
}
+/* Print summary info about the state of the optimized allocator */
+void
+_PyMethod_DebugMallocStats(FILE *out)
+{
+ _PyDebugAllocatorStats(out,
+ "free PyMethodObject",
+ numfree, sizeof(PyMethodObject));
+}
+
/* ------------------------------------------------------------------------
* instance method
*/
diff --git a/Objects/dictobject.c b/Objects/dictobject.c
--- a/Objects/dictobject.c
+++ b/Objects/dictobject.c
@@ -255,6 +255,15 @@
return ret;
}
+/* Print summary info about the state of the optimized allocator */
+void
+_PyDict_DebugMallocStats(FILE *out)
+{
+ _PyDebugAllocatorStats(out,
+ "free PyDictObject", numfree, sizeof(PyDictObject));
+}
+
+
void
PyDict_Fini(void)
{
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -1933,6 +1933,16 @@
(void)PyFloat_ClearFreeList();
}
+/* Print summary info about the state of the optimized allocator */
+void
+_PyFloat_DebugMallocStats(FILE *out)
+{
+ _PyDebugAllocatorStats(out,
+ "free PyFloatObject",
+ numfree, sizeof(PyFloatObject));
+}
+
+
/*----------------------------------------------------------------------------
* _PyFloat_{Pack,Unpack}{4,8}. See floatobject.h.
*/
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -955,3 +955,13 @@
Py_XDECREF(builtin_object);
builtin_object = NULL;
}
+
+/* Print summary info about the state of the optimized allocator */
+void
+_PyFrame_DebugMallocStats(FILE *out)
+{
+ _PyDebugAllocatorStats(out,
+ "free PyFrameObject",
+ numfree, sizeof(PyFrameObject));
+}
+
diff --git a/Objects/listobject.c b/Objects/listobject.c
--- a/Objects/listobject.c
+++ b/Objects/listobject.c
@@ -117,6 +117,15 @@
PyList_ClearFreeList();
}
+/* Print summary info about the state of the optimized allocator */
+void
+_PyList_DebugMallocStats(FILE *out)
+{
+ _PyDebugAllocatorStats(out,
+ "free PyListObject",
+ numfree, sizeof(PyListObject));
+}
+
PyObject *
PyList_New(Py_ssize_t size)
{
diff --git a/Objects/methodobject.c b/Objects/methodobject.c
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -338,6 +338,15 @@
(void)PyCFunction_ClearFreeList();
}
+/* Print summary info about the state of the optimized allocator */
+void
+_PyCFunction_DebugMallocStats(FILE *out)
+{
+ _PyDebugAllocatorStats(out,
+ "free PyCFunction",
+ numfree, sizeof(PyCFunction));
+}
+
/* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(),
but it's part of the API so we need to keep a function around that
existing C extensions can call.
diff --git a/Objects/object.c b/Objects/object.c
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -1852,6 +1852,18 @@
PyMem_FREE(p);
}
+void
+_PyObject_DebugTypeStats(FILE *out)
+{
+ _PyCFunction_DebugMallocStats(out);
+ _PyDict_DebugMallocStats(out);
+ _PyFloat_DebugMallocStats(out);
+ _PyFrame_DebugMallocStats(out);
+ _PyList_DebugMallocStats(out);
+ _PyMethod_DebugMallocStats(out);
+ _PySet_DebugMallocStats(out);
+ _PyTuple_DebugMallocStats(out);
+}
/* These methods are used to control infinite recursion in repr, str, print,
etc. Container objects that may recursively contain themselves,
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -523,12 +523,10 @@
/* Number of arenas allocated that haven't been free()'d. */
static size_t narenas_currently_allocated = 0;
-#ifdef PYMALLOC_DEBUG
/* Total number of times malloc() called to allocate an arena. */
static size_t ntimes_arena_allocated = 0;
/* High water mark (max value ever seen) for narenas_currently_allocated. */
static size_t narenas_highwater = 0;
-#endif
/* Allocate a new arena. If we run out of memory, return NULL. Else
* allocate a new arena, and return the address of an arena_object
@@ -545,7 +543,7 @@
#ifdef PYMALLOC_DEBUG
if (Py_GETENV("PYTHONMALLOCSTATS"))
- _PyObject_DebugMallocStats();
+ _PyObject_DebugMallocStats(stderr);
#endif
if (unused_arena_objects == NULL) {
uint i;
@@ -613,11 +611,9 @@
arenaobj->address = (uptr)address;
++narenas_currently_allocated;
-#ifdef PYMALLOC_DEBUG
++ntimes_arena_allocated;
if (narenas_currently_allocated > narenas_highwater)
narenas_highwater = narenas_currently_allocated;
-#endif
arenaobj->freepools = NULL;
/* pool_address <- first pool-aligned address in the arena
nfreepools <- number of whole pools that fit after alignment */
@@ -1723,17 +1719,19 @@
}
}
+#endif /* PYMALLOC_DEBUG */
+
static size_t
-printone(const char* msg, size_t value)
+printone(FILE *out, const char* msg, size_t value)
{
int i, k;
char buf[100];
size_t origvalue = value;
- fputs(msg, stderr);
+ fputs(msg, out);
for (i = (int)strlen(msg); i < 35; ++i)
- fputc(' ', stderr);
- fputc('=', stderr);
+ fputc(' ', out);
+ fputc('=', out);
/* Write the value with commas. */
i = 22;
@@ -1754,17 +1752,33 @@
while (i >= 0)
buf[i--] = ' ';
- fputs(buf, stderr);
+ fputs(buf, out);
return origvalue;
}
-/* Print summary info to stderr about the state of pymalloc's structures.
+void
+_PyDebugAllocatorStats(FILE *out,
+ const char *block_name, int num_blocks, size_t sizeof_block)
+{
+ char buf1[128];
+ char buf2[128];
+ PyOS_snprintf(buf1, sizeof(buf1),
+ "%d %ss * %zd bytes each",
+ num_blocks, block_name, sizeof_block);
+ PyOS_snprintf(buf2, sizeof(buf2),
+ "%48s ", buf1);
+ (void)printone(out, buf2, num_blocks * sizeof_block);
+}
+
+#ifdef WITH_PYMALLOC
+
+/* Print summary info to "out" about the state of pymalloc's structures.
* In Py_DEBUG mode, also perform some expensive internal consistency
* checks.
*/
void
-_PyObject_DebugMallocStats(void)
+_PyObject_DebugMallocStats(FILE *out)
{
uint i;
const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT;
@@ -1793,7 +1807,7 @@
size_t total;
char buf[128];
- fprintf(stderr, "Small block threshold = %d, in %u size classes.\n",
+ fprintf(out, "Small block threshold = %d, in %u size classes.\n",
SMALL_REQUEST_THRESHOLD, numclasses);
for (i = 0; i < numclasses; ++i)
@@ -1847,10 +1861,10 @@
}
assert(narenas == narenas_currently_allocated);
- fputc('\n', stderr);
+ fputc('\n', out);
fputs("class size num pools blocks in use avail blocks\n"
"----- ---- --------- ------------- ------------\n",
- stderr);
+ out);
for (i = 0; i < numclasses; ++i) {
size_t p = numpools[i];
@@ -1861,7 +1875,7 @@
assert(b == 0 && f == 0);
continue;
}
- fprintf(stderr, "%5u %6u "
+ fprintf(out, "%5u %6u "
"%11" PY_FORMAT_SIZE_T "u "
"%15" PY_FORMAT_SIZE_T "u "
"%13" PY_FORMAT_SIZE_T "u\n",
@@ -1871,35 +1885,36 @@
pool_header_bytes += p * POOL_OVERHEAD;
quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size);
}
- fputc('\n', stderr);
- (void)printone("# times object malloc called", serialno);
-
- (void)printone("# arenas allocated total", ntimes_arena_allocated);
- (void)printone("# arenas reclaimed", ntimes_arena_allocated - narenas);
- (void)printone("# arenas highwater mark", narenas_highwater);
- (void)printone("# arenas allocated current", narenas);
+ fputc('\n', out);
+#ifdef PYMALLOC_DEBUG
+ (void)printone(out, "# times object malloc called", serialno);
+#endif
+ (void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
+ (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
+ (void)printone(out, "# arenas highwater mark", narenas_highwater);
+ (void)printone(out, "# arenas allocated current", narenas);
PyOS_snprintf(buf, sizeof(buf),
"%" PY_FORMAT_SIZE_T "u arenas * %d bytes/arena",
narenas, ARENA_SIZE);
- (void)printone(buf, narenas * ARENA_SIZE);
+ (void)printone(out, buf, narenas * ARENA_SIZE);
- fputc('\n', stderr);
+ fputc('\n', out);
- total = printone("# bytes in allocated blocks", allocated_bytes);
- total += printone("# bytes in available blocks", available_bytes);
+ total = printone(out, "# bytes in allocated blocks", allocated_bytes);
+ total += printone(out, "# bytes in available blocks", available_bytes);
PyOS_snprintf(buf, sizeof(buf),
"%u unused pools * %d bytes", numfreepools, POOL_SIZE);
- total += printone(buf, (size_t)numfreepools * POOL_SIZE);
+ total += printone(out, buf, (size_t)numfreepools * POOL_SIZE);
- total += printone("# bytes lost to pool headers", pool_header_bytes);
- total += printone("# bytes lost to quantization", quantization);
- total += printone("# bytes lost to arena alignment", arena_alignment);
- (void)printone("Total", total);
+ total += printone(out, "# bytes lost to pool headers", pool_header_bytes);
+ total += printone(out, "# bytes lost to quantization", quantization);
+ total += printone(out, "# bytes lost to arena alignment", arena_alignment);
+ (void)printone(out, "Total", total);
}
-#endif /* PYMALLOC_DEBUG */
+#endif /* #ifdef WITH_PYMALLOC */
#ifdef Py_USING_MEMORY_DEBUGGER
/* Make this function last so gcc won't inline it since the definition is
diff --git a/Objects/setobject.c b/Objects/setobject.c
--- a/Objects/setobject.c
+++ b/Objects/setobject.c
@@ -1133,6 +1133,16 @@
Py_CLEAR(emptyfrozenset);
}
+/* Print summary info about the state of the optimized allocator */
+void
+_PySet_DebugMallocStats(FILE *out)
+{
+ _PyDebugAllocatorStats(out,
+ "free PySetObject",
+ numfree, sizeof(PySetObject));
+}
+
+
static PyObject *
set_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -45,6 +45,22 @@
}
#endif
+/* Print summary info about the state of the optimized allocator */
+void
+_PyTuple_DebugMallocStats(FILE *out)
+{
+#if PyTuple_MAXSAVESIZE > 0
+ int i;
+ char buf[128];
+ for (i = 1; i < PyTuple_MAXSAVESIZE; i++) {
+ PyOS_snprintf(buf, sizeof(buf),
+ "free %d-sized PyTupleObject", i);
+ _PyDebugAllocatorStats(out,
+ buf,
+ numfree[i], _PyObject_VAR_SIZE(&PyTuple_Type, i));
+ }
+#endif
+}
PyObject *
PyTuple_New(register Py_ssize_t size)
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -642,7 +642,7 @@
#endif /* Py_TRACE_REFS */
#ifdef PYMALLOC_DEBUG
if (Py_GETENV("PYTHONMALLOCSTATS"))
- _PyObject_DebugMallocStats();
+ _PyObject_DebugMallocStats(stderr);
#endif
call_ll_exitfuncs();
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -997,6 +997,27 @@
extern "C" {
#endif
+static PyObject *
+sys_debugmallocstats(PyObject *self, PyObject *args)
+{
+#ifdef WITH_PYMALLOC
+ _PyObject_DebugMallocStats(stderr);
+ fputc('\n', stderr);
+#endif
+ _PyObject_DebugTypeStats(stderr);
+
+ Py_RETURN_NONE;
+}
+PyDoc_STRVAR(debugmallocstats_doc,
+"_debugmallocstats()\n\
+\n\
+Print summary info to stderr about the state of\n\
+pymalloc's structures.\n\
+\n\
+In Py_DEBUG mode, also perform some expensive internal consistency\n\
+checks.\n\
+");
+
#ifdef Py_TRACE_REFS
/* Defined in objects.c because it uses static globals if that file */
extern PyObject *_Py_GetObjects(PyObject *, PyObject *);
@@ -1093,6 +1114,8 @@
{"settrace", sys_settrace, METH_O, settrace_doc},
{"gettrace", sys_gettrace, METH_NOARGS, gettrace_doc},
{"call_tracing", sys_call_tracing, METH_VARARGS, call_tracing_doc},
+ {"_debugmallocstats", sys_debugmallocstats, METH_VARARGS,
+ debugmallocstats_doc},
{NULL, NULL} /* sentinel */
};
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list