[Python-checkins] r60567 - in python/trunk: Doc/c-api/float.rst Doc/c-api/int.rst Doc/library/sys.rst Include/floatobject.h Include/intobject.h Lib/test/regrtest.py Lib/test/test_sys.py Misc/NEWS Objects/floatobject.c Objects/intobject.c Python/sysmodule.c
christian.heimes
python-checkins at python.org
Mon Feb 4 19:00:13 CET 2008
Author: christian.heimes
Date: Mon Feb 4 19:00:12 2008
New Revision: 60567
Modified:
python/trunk/Doc/c-api/float.rst
python/trunk/Doc/c-api/int.rst
python/trunk/Doc/library/sys.rst
python/trunk/Include/floatobject.h
python/trunk/Include/intobject.h
python/trunk/Lib/test/regrtest.py
python/trunk/Lib/test/test_sys.py
python/trunk/Misc/NEWS
python/trunk/Objects/floatobject.c
python/trunk/Objects/intobject.c
python/trunk/Python/sysmodule.c
Log:
Patch #1953
I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers.
The patch also renames sys._cleartypecache to sys._clear_type_cache
Modified: python/trunk/Doc/c-api/float.rst
==============================================================================
--- python/trunk/Doc/c-api/float.rst (original)
+++ python/trunk/Doc/c-api/float.rst Mon Feb 4 19:00:12 2008
@@ -84,3 +84,12 @@
Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`.
.. versionadded:: 2.6
+
+
+.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
+
+ Compact the float free list. *bc* is the number of allocated blocks before
+ blocks are freed, *bf* is the number of freed blocks and *sum* is the number
+ of remaining objects in the blocks.
+
+ .. versionadded:: 2.6
Modified: python/trunk/Doc/c-api/int.rst
==============================================================================
--- python/trunk/Doc/c-api/int.rst (original)
+++ python/trunk/Doc/c-api/int.rst Mon Feb 4 19:00:12 2008
@@ -120,3 +120,12 @@
Return the system's idea of the largest integer it can handle
(:const:`LONG_MAX`, as defined in the system header files).
+
+
+.. cfunction:: void PyInt_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
+
+ Compact the integer free list. *bc* is the number of allocated blocks before
+ blocks are freed, *bf* is the number of freed blocks and *sum* is the number
+ of remaining objects in the blocks.
+
+ .. versionadded:: 2.6
Modified: python/trunk/Doc/library/sys.rst
==============================================================================
--- python/trunk/Doc/library/sys.rst (original)
+++ python/trunk/Doc/library/sys.rst Mon Feb 4 19:00:12 2008
@@ -58,9 +58,29 @@
A string containing the copyright pertaining to the Python interpreter.
-.. function:: _cleartypecache()
+.. function:: _compact_freelists()
- Clear the internal type lookup cache.
+ Compact the free lists of integers and floats by deallocating unused blocks.
+ It can reduce the memory usage of the Python process several tenth of
+ thousands of integers or floats have been allocated at once.
+
+ The return value is a tuple of tuples each containing three elements,
+ amount of used objects, total block count before the blocks are deallocated
+ and amount of freed blocks. The first tuple refers to ints, the second to
+ floats.
+
+ This function should be used for specialized purposes only.
+
+ .. versionadded:: 2.6
+
+
+.. function:: _clear_type_cache()
+
+ Clear the internal type cache. The type cache is used to speed up attribute
+ and method lookups. Use the function *only* to drop unnecessary references
+ during reference leak debugging.
+
+ This function should be used for internal and specialized purposes only.
.. versionadded:: 2.6
Modified: python/trunk/Include/floatobject.h
==============================================================================
--- python/trunk/Include/floatobject.h (original)
+++ python/trunk/Include/floatobject.h Mon Feb 4 19:00:12 2008
@@ -101,6 +101,8 @@
PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le);
PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
+/* free list api */
+PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *);
#ifdef __cplusplus
}
Modified: python/trunk/Include/intobject.h
==============================================================================
--- python/trunk/Include/intobject.h (original)
+++ python/trunk/Include/intobject.h Mon Feb 4 19:00:12 2008
@@ -59,6 +59,9 @@
PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int);
PyAPI_FUNC(long) PyOS_strtol(char *, char **, int);
+/* free list api */
+PyAPI_FUNC(void) PyInt_CompactFreeList(size_t *, size_t *, size_t *);
+
#ifdef __cplusplus
}
#endif
Modified: python/trunk/Lib/test/regrtest.py
==============================================================================
--- python/trunk/Lib/test/regrtest.py (original)
+++ python/trunk/Lib/test/regrtest.py Mon Feb 4 19:00:12 2008
@@ -710,7 +710,7 @@
sys.path_importer_cache.update(pic)
# clear type cache
- sys._cleartypecache()
+ sys._clear_type_cache()
# Clear ABC registries, restoring previously saved ABC registries.
for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:
Modified: python/trunk/Lib/test/test_sys.py
==============================================================================
--- python/trunk/Lib/test/test_sys.py (original)
+++ python/trunk/Lib/test/test_sys.py Mon Feb 4 19:00:12 2008
@@ -363,6 +363,24 @@
self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
self.assert_(repr(sys.flags))
+ def test_clear_type_cache(self):
+ sys._clear_type_cache()
+
+ def test_compact_freelists(self):
+ sys._compact_freelists()
+ r = sys._compact_freelists()
+ # freed blocks shouldn't change
+ self.assertEqual(r[0][2], 0)
+ self.assertEqual(r[1][2], 0)
+ # fill freelists
+ ints = list(range(12000))
+ floats = [float(i) for i in ints]
+ del ints
+ del floats
+ # should free more than 200 blocks each
+ r = sys._compact_freelists()
+ self.assert_(r[0][2] > 200, r[0][2])
+ self.assert_(r[1][2] > 200, r[1][2])
def test_main():
test.test_support.run_unittest(SysModuleTest)
Modified: python/trunk/Misc/NEWS
==============================================================================
--- python/trunk/Misc/NEWS (original)
+++ python/trunk/Misc/NEWS Mon Feb 4 19:00:12 2008
@@ -12,6 +12,10 @@
Core and builtins
-----------------
+- Patch #1953: Added ´´sys._compact_freelists()´´ and the C API functions
+ ´´PyInt_CompactFreeList´´ and ´´PyFloat_CompactFreeList´´
+ to compact the internal free lists of pre-allocted ints and floats.
+
- Bug #1983: Fixed return type of fork(), fork1() and forkpty() calls.
Python expected the return type int but the fork familie returns pi_t.
@@ -21,7 +25,7 @@
- Patch #1970 by Antoine Pitrou: Speedup unicode whitespace and linebreak
detection
-- Added ``PyType_ClearCache()`` and ``sys._cleartypecache`` to clear the
+- Added ``PyType_ClearCache()`` and ``sys._clear_type_cache`` to clear the
internal lookup cache for ref leak tests.
- Patch #1473257: generator objects gain a gi_code attribute. This is the
Modified: python/trunk/Objects/floatobject.c
==============================================================================
--- python/trunk/Objects/floatobject.c (original)
+++ python/trunk/Objects/floatobject.c Mon Feb 4 19:00:12 2008
@@ -1609,17 +1609,15 @@
}
void
-PyFloat_Fini(void)
+PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
{
PyFloatObject *p;
PyFloatBlock *list, *next;
unsigned i;
- int bc, bf; /* block count, number of freed blocks */
- int frem, fsum; /* remaining unfreed floats per block, total */
+ size_t bc = 0, bf = 0; /* block count, number of freed blocks */
+ size_t fsum = 0; /* total unfreed ints */
+ int frem; /* remaining unfreed ints per block */
- bc = 0;
- bf = 0;
- fsum = 0;
list = block_list;
block_list = NULL;
free_list = NULL;
@@ -1654,6 +1652,22 @@
fsum += frem;
list = next;
}
+ *pbc = bc;
+ *pbf = bf;
+ *bsum = fsum;
+}
+
+void
+PyFloat_Fini(void)
+{
+ PyFloatObject *p;
+ PyFloatBlock *list;
+ unsigned i;
+ size_t bc, bf; /* block count, number of freed blocks */
+ size_t fsum; /* total unfreed floats per block */
+
+ PyFloat_CompactFreeList(&bc, &bf, &fsum);
+
if (!Py_VerboseFlag)
return;
fprintf(stderr, "# cleanup floats");
@@ -1662,7 +1676,9 @@
}
else {
fprintf(stderr,
- ": %d unfreed float%s in %d out of %d block%s\n",
+ ": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %"
+ PY_FORMAT_SIZE_T "d out of %"
+ PY_FORMAT_SIZE_T "d block%s\n",
fsum, fsum == 1 ? "" : "s",
bc - bf, bc, bc == 1 ? "" : "s");
}
Modified: python/trunk/Objects/intobject.c
==============================================================================
--- python/trunk/Objects/intobject.c (original)
+++ python/trunk/Objects/intobject.c Mon Feb 4 19:00:12 2008
@@ -1202,28 +1202,15 @@
}
void
-PyInt_Fini(void)
+PyInt_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
{
PyIntObject *p;
PyIntBlock *list, *next;
- int i;
unsigned int ctr;
- int bc, bf; /* block count, number of freed blocks */
- int irem, isum; /* remaining unfreed ints per block, total */
+ size_t bc = 0, bf = 0; /* block count, number of freed blocks */
+ size_t isum = 0; /* total unfreed ints */
+ int irem; /* remaining unfreed ints per block */
-#if NSMALLNEGINTS + NSMALLPOSINTS > 0
- PyIntObject **q;
-
- i = NSMALLNEGINTS + NSMALLPOSINTS;
- q = small_ints;
- while (--i >= 0) {
- Py_XDECREF(*q);
- *q++ = NULL;
- }
-#endif
- bc = 0;
- bf = 0;
- isum = 0;
list = block_list;
block_list = NULL;
free_list = NULL;
@@ -1268,6 +1255,33 @@
isum += irem;
list = next;
}
+
+ *pbc = bc;
+ *pbf = bf;
+ *bsum = isum;
+}
+
+void
+PyInt_Fini(void)
+{
+ PyIntObject *p;
+ PyIntBlock *list;
+ unsigned int ctr;
+ size_t bc, bf; /* block count, number of freed blocks */
+ size_t isum; /* total unfreed ints per block */
+
+#if NSMALLNEGINTS + NSMALLPOSINTS > 0
+ int i;
+ PyIntObject **q;
+
+ i = NSMALLNEGINTS + NSMALLPOSINTS;
+ q = small_ints;
+ while (--i >= 0) {
+ Py_XDECREF(*q);
+ *q++ = NULL;
+ }
+#endif
+ PyInt_CompactFreeList(&bc, &bf, &isum);
if (!Py_VerboseFlag)
return;
fprintf(stderr, "# cleanup ints");
@@ -1276,7 +1290,9 @@
}
else {
fprintf(stderr,
- ": %d unfreed int%s in %d out of %d block%s\n",
+ ": %" PY_FORMAT_SIZE_T "d unfreed ints%s in %"
+ PY_FORMAT_SIZE_T "d out of %"
+ PY_FORMAT_SIZE_T "d block%s\n",
isum, isum == 1 ? "" : "s",
bc - bf, bc, bc == 1 ? "" : "s");
}
Modified: python/trunk/Python/sysmodule.c
==============================================================================
--- python/trunk/Python/sysmodule.c (original)
+++ python/trunk/Python/sysmodule.c Mon Feb 4 19:00:12 2008
@@ -754,17 +754,6 @@
10. Number of stack pops performed by call_function()"
);
-static PyObject *
-sys_cleartypecache(PyObject* self, PyObject* args)
-{
- PyType_ClearCache();
- Py_RETURN_NONE;
-}
-
-PyDoc_STRVAR(cleartypecache_doc,
-"_cleartypecache() -> None\n\
-Clear the internal type lookup cache.");
-
#ifdef __cplusplus
extern "C" {
#endif
@@ -783,12 +772,44 @@
}
#endif
+static PyObject *
+sys_clear_type_cache(PyObject* self, PyObject* args)
+{
+ PyType_ClearCache();
+ Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(sys_clear_type_cache__doc__,
+"_clear_type_cache() -> None\n\
+Clear the internal type lookup cache.");
+
+
+static PyObject *
+sys_compact_freelists(PyObject* self, PyObject* args)
+{
+ size_t isum, ibc, ibf;
+ size_t fsum, fbc, fbf;
+
+ PyInt_CompactFreeList(&ibc, &ibf, &isum);
+ PyFloat_CompactFreeList(&fbc, &fbf, &fsum);
+
+ return Py_BuildValue("(kkk)(kkk)", isum, ibc, ibf,
+ fsum, fbc, fbf);
+
+}
+
+PyDoc_STRVAR(sys_compact_freelists__doc__,
+"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks), ...)\n\
+Compact the free lists of ints and floats.");
+
static PyMethodDef sys_methods[] = {
/* Might as well keep this in alphabetic order */
{"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS,
callstats_doc},
- {"_cleartypecache", sys_cleartypecache, METH_NOARGS,
- cleartypecache_doc},
+ {"_clear_type_cache", sys_clear_type_cache, METH_NOARGS,
+ sys_clear_type_cache__doc__},
+ {"_compact_freelists", sys_compact_freelists, METH_NOARGS,
+ sys_compact_freelists__doc__},
{"_current_frames", sys_current_frames, METH_NOARGS,
current_frames_doc},
{"displayhook", sys_displayhook, METH_O, displayhook_doc},
More information about the Python-checkins
mailing list