[Python-checkins] peps: PEP 0445: examples
victor.stinner
python-checkins at python.org
Tue Jun 18 01:02:34 CEST 2013
http://hg.python.org/peps/rev/9f1d6908bc70
changeset: 4935:9f1d6908bc70
user: Victor Stinner <victor.stinner at gmail.com>
date: Tue Jun 18 01:02:16 2013 +0200
summary:
PEP 0445: examples
files:
pep-0445.txt | 169 ++++++++++++++++++++---
pep-0445/alloc_hooks.c | 114 ++++++++++++++++
pep-0445/replace_allocs.c | 44 ++++++
pep-0445/replace_pymalloc.c | 34 ++++
4 files changed, 339 insertions(+), 22 deletions(-)
diff --git a/pep-0445.txt b/pep-0445.txt
--- a/pep-0445.txt
+++ b/pep-0445.txt
@@ -40,25 +40,32 @@
Proposal
========
+API changes
+-----------
+
* Add a new ``PyMemAllocators`` structure
* Add new GIL-free memory allocator functions:
- - ``PyMem_RawMalloc()``
- - ``PyMem_RawRealloc()``
- - ``PyMem_RawFree()``
+ - ``void* PyMem_RawMalloc(size_t size)``
+ - ``void* PyMem_RawRealloc(void *ptr, size_t new_size)``
+ - ``void PyMem_RawFree(void *ptr)``
* Add new functions to get and set memory allocators:
- - ``PyMem_GetRawAllocators()``, ``PyMem_SetRawAllocators()``
- - ``PyMem_GetAllocators()``, ``PyMem_SetAllocators()``
- - ``PyObject_GetAllocators()``, ``PyObject_SetAllocators()``
- - ``_PyObject_GetArenaAllocators()``, ``_PyObject_SetArenaAllocators()``
+ - ``void PyMem_GetRawAllocators(PyMemAllocators *allocators)``
+ - ``void PyMem_SetRawAllocators(PyMemAllocators *allocators)``
+ - ``void PyMem_GetAllocators(PyMemAllocators *allocators)``
+ - ``void PyMem_SetAllocators(PyMemAllocators *allocators)``
+ - ``void PyObject_GetAllocators(PyMemAllocators *allocators)``
+ - ``void PyObject_SetAllocators(PyMemAllocators *allocators)``
+ - ``void _PyObject_GetArenaAllocators(void **ctx_p, void* (**malloc_p) (void *ctx, size_t size), void (**free_p) (void *ctx, void *ptr, size_t size))``
+ - ``void _PyObject_SetArenaAllocators(void *ctx, void* (*malloc) (void *ctx, size_t size), void (*free) (void *ctx, void *ptr, size_t size))``
* Add a new function to setup debug hooks after memory allocators were
replaced:
- - ``PyMem_SetupDebugHooks()``
+ - ``void PyMem_SetupDebugHooks(void)``
* ``PyMem_Malloc()`` and ``PyMem_Realloc()`` now always call ``malloc()`` and
``realloc()``, instead of calling ``PyObject_Malloc()`` and
@@ -70,16 +77,140 @@
``realloc()``
+Examples
+========
+
+Use case 1: replace memory allocators, keeping pymalloc
+-------------------------------------------------------
+
+Setup your custom memory allocators, keeping pymalloc::
+
+ /* global variable, don't use a variable allocated on the stack! */
+ int magic = 42;
+
+ int my_malloc(void *ctx, size_t size);
+ int my_realloc(void *ctx, void *ptr, size_t new_size);
+ void my_free(void *ctx, void *ptr);
+
+ int my_alloc_arena(void *ctx, size_t size);
+ int my_free_arena(void *ctx, void *ptr, size_t size);
+
+ void setup_custom_allocators(void)
+ {
+ PyMemAllocators alloc;
+ alloc.ctx = &magic;
+ alloc.malloc = my_malloc;
+ alloc.realloc = my_realloc;
+ alloc.free = my_free;
+
+ PyMem_SetRawAllocators(&alloc);
+ PyMem_SetAllocators(&alloc);
+ _PyObject_SetArenaAllocators(&magic, my_alloc_arena, my_free_arena);
+
+ PyMem_SetupDebugHooks();
+ }
+
+.. warning::
+ Remove call ``PyMem_SetRawAllocators(&alloc);`` if the new allocators are
+ not thread-safe.
+
+Full example:
+`replace_allocs.c <http://hg.python.org/peps/file/tip/pep-0445/replace_allocs.c>`_.
+
+
+Use case 2: replace memory allocators, overriding pymalloc
+----------------------------------------------------------
+
+If your allocator is optimized for allocation of small objects (less than 512
+bytes) with a short liftime, you can replace override pymalloc (replace
+``PyObject_Malloc()``). Example::
+
+ /* global variable, don't use a variable allocated on the stack! */
+ int magic = 42;
+
+ int my_malloc(void *ctx, size_t size);
+ int my_realloc(void *ctx, void *ptr, size_t new_size);
+ void my_free(void *ctx, void *ptr);
+
+ void setup_custom_allocators(void)
+ {
+ PyMemAllocators alloc;
+ alloc.ctx = &magic;
+ alloc.malloc = my_malloc;
+ alloc.realloc = my_realloc;
+ alloc.free = my_free;
+
+ PyMem_SetRawAllocators(&alloc);
+ PyMem_SetAllocators(&alloc);
+ PyObject_SetAllocators(&areana);
+ PyMem_SetupDebugHooks();
+ }
+
+Full example:
+`replace_pymalloc.c <http://hg.python.org/peps/file/tip/pep-0445/replace_pymalloc.c>`_.
+
+
+Use case 3: hook allocators
+---------------------------
+
+Setup hooks on memory allocators::
+
+ /* global variable, don't use a variable allocated on the stack! */
+ struct {
+ PyMemAllocators pymem;
+ PyMemAllocators pymem_raw;
+ PyMemAllocators pyobj;
+ int magic;
+ } hook;
+
+ int hook_malloc(void *ctx, size_t size);
+ int hook_realloc(void *ctx, void *ptr, size_t new_size);
+ void hook_free(void *ctx, void *ptr);
+
+ /* Must be called before the first allocation, or hook_realloc() and
+ hook_free() will crash */
+ void setup_custom_allocators(void)
+ {
+ PyMemAllocators alloc;
+
+ alloc.ctx = &magic;
+ alloc.malloc = hook_malloc;
+ alloc.realloc = hook_realloc;
+ alloc.free = hook_free;
+
+ PyMem_GetRawAllocators(&alloc.pymem_raw);
+ alloc.ctx = &alloc.pymem_raw;
+ PyMem_SetRawAllocators(&alloc);
+
+ PyMem_GetAllocators(&alloc.pymem);
+ alloc.ctx = &alloc.pymem;
+ PyMem_SetAllocators(&alloc);
+
+ PyObject_GetAllocators(&alloc.pyobj);
+ alloc.ctx = &alloc.pyobj;
+ PyObject_SetAllocators(&alloc);
+ }
+
+.. note::
+ No need to call ``PyMem_SetupDebugHooks()``: it is already installed by
+ default.
+
+Full example tracking memory usage:
+`alloc_hooks.c <http://hg.python.org/peps/file/tip/pep-0445/alloc_hooks.c>`_.
+
+
+
Performances
============
-The Python benchmarks suite (-b 2n3): some tests are 1.04x faster, some tests
-are 1.04 slower, significant is between 115 and -191. I don't understand these
-output, but I guess that the overhead cannot be seen with such test.
+The `Python benchmarks suite <http://hg.python.org/benchmarks>`_ (-b 2n3): some
+tests are 1.04x faster, some tests are 1.04 slower, significant is between 115
+and -191. I don't understand these output, but I guess that the overhead cannot
+be seen with such test.
pybench: "+0.1%" (diff between -4.9% and +5.6%).
-Full output attached to the issue #3329.
+The full output is attached to the issue #3329.
Alternatives
@@ -163,6 +294,10 @@
* glib: `g_mem_set_vtable()
<http://developer.gnome.org/glib/unstable/glib-Memory-Allocation.html#g-mem-set-vtable>`_
+See also the `GNU libc: Memory Allocation Hooks
+<http://www.gnu.org/software/libc/manual/html_node/Hooks-for-Malloc.html>`_.
+
+
Memory allocators
=================
@@ -225,13 +360,3 @@
* `PySizer (developed for Python 2.4)
<http://pysizer.8325.org/>`_
-APIs to set a custom memory allocator and/or hook memory allocators:
-
-* `GNU libc: Memory Allocation Hooks
- <http://www.gnu.org/software/libc/manual/html_node/Hooks-for-Malloc.html>`_
-
-Other:
-
-* `Python benchmark suite
- <http://hg.python.org/benchmarks>`_
-
diff --git a/pep-0445/alloc_hooks.c b/pep-0445/alloc_hooks.c
new file mode 100644
--- /dev/null
+++ b/pep-0445/alloc_hooks.c
@@ -0,0 +1,114 @@
+/* global variable, don't use a variable allocated on the stack! */
+struct {
+ PyMemAllocators pymem;
+ PyMemAllocators pymem_raw;
+ PyMemAllocators pyobj;
+ size_t allocated;
+} hook;
+
+/* read_size_t() and write_size_t() are not needed if malloc() and realloc()
+ always return a pointer aligned to sizeof(size_t) bytes */
+static size_t read_size_t(const void *p)
+{
+ const unsigned char *q = (const unsigned char *)p;
+ size_t result = *q++;
+ int i;
+
+ for (i = SST; --i > 0; ++q)
+ result = (result << 8) | *q;
+ return result;
+}
+
+static void write_size_t(void *p, size_t n)
+{
+ unsigned char *q = (unsigned char *)p + SST - 1;
+ int i;
+
+ for (i = SST; --i >= 0; --q) {
+ *q = (unsigned char)(n & 0xff);
+ n >>= 8;
+ }
+}
+
+static int hook_malloc(void *ctx, size_t size)
+{
+ PyMemAllocators *alloc;
+ char *ptr;
+
+ size += sizeof(size_t);
+ ptr = alloc->malloc(size);
+ if (ptr != NULL) {
+ write_size_t(ptr, size);
+ ptr += sizeof(size_t);
+ allocated += size;
+ }
+ return ptr;
+}
+
+static int hook_realloc(void *ctx, void *void_ptr, size_t new_size)
+{
+ PyMemAllocators *alloc;
+ char *ptr, *ptr2;
+ size_t old_size;
+
+ ptr = void_ptr;
+ if (ptr) {
+ ptr -= sizeof(size_t);
+ old_size = read_size_t(ptr);
+ }
+ else {
+ old_size = 0;
+ }
+
+ ptr2 = alloc->realloc(ptr, size);
+ if (ptr2 != NULL) {
+ write_size_t(ptr2, size);
+ ptr2 += sizeof(size_t);
+ allocated -= old_size;
+ allocated += new_size;
+ }
+ return ptr2;
+}
+
+static void hook_free(void *ctx, void *void_ptr)
+{
+ PyMemAllocators *alloc;
+ char *ptr;
+ size_t size;
+
+ ptr = void_ptr;
+ if (!ptr)
+ return;
+
+ ptr -= sizeof(size_t);
+ size = read_size_t(ptr);
+
+ alloc->free(ptr);
+ allocated -= size;
+}
+
+/* Must be called before the first allocation, or hook_realloc() and
+ hook_free() will crash */
+void setup_custom_allocators(void)
+{
+ PyMemAllocators alloc;
+
+ alloc.malloc = my_malloc;
+ alloc.realloc = my_realloc;
+ alloc.free = my_free;
+
+ /* disabled: the hook is not thread-safe */
+#if 0
+ PyMem_GetRawAllocators(&alloc.pymem_raw);
+ alloc.ctx = &alloc.pymem_raw;
+ PyMem_SetRawAllocators(&alloc);
+#endif
+
+ PyMem_GetAllocators(&alloc.pymem);
+ alloc.ctx = &alloc.pymem;
+ PyMem_SetAllocators(&alloc);
+
+ PyObject_GetAllocators(&alloc.pyobj);
+ alloc.ctx = &alloc.pyobj;
+ PyObject_SetAllocators(&alloc);
+}
diff --git a/pep-0445/replace_allocs.c b/pep-0445/replace_allocs.c
new file mode 100644
--- /dev/null
+++ b/pep-0445/replace_allocs.c
@@ -0,0 +1,44 @@
+#include <stdlib.h>
+
+/* global variable, don't use a variable allocated on the stack! */
+int magic = 42;
+
+int my_malloc(void *ctx, size_t size)
+{
+ return malloc(size);
+}
+
+int my_realloc(void *ctx, void *ptr, size_t new_size)
+{
+ return realloc(ptr, new_size);
+}
+
+void my_free(void *ctx, void *ptr)
+{
+ free(ptr);
+}
+
+int my_alloc_arena(void *ctx, size_t size)
+{
+ return malloc(size);
+}
+
+int my_free_arena(void *ctx, void *ptr, size_t size)
+{
+ free(ptr);
+}
+
+void setup_custom_allocators(void)
+{
+ PyMemAllocators alloc;
+ alloc.ctx = &magic;
+ alloc.malloc = my_malloc;
+ alloc.realloc = my_realloc;
+ alloc.free = my_free;
+
+ PyMem_SetRawAllocators(&alloc);
+ PyMem_SetAllocators(&alloc);
+ _PyObject_SetArenaAllocators(&magic, my_alloc_arena, my_free_arena);
+ PyMem_SetupDebugHooks();
+}
+
diff --git a/pep-0445/replace_pymalloc.c b/pep-0445/replace_pymalloc.c
new file mode 100644
--- /dev/null
+++ b/pep-0445/replace_pymalloc.c
@@ -0,0 +1,34 @@
+#include <stdlib.h>
+
+/* global variable, don't use a variable allocated on the stack! */
+int magic = 42;
+
+int my_malloc(void *ctx, size_t size)
+{
+ return malloc(size);
+}
+
+int my_realloc(void *ctx, void *ptr, size_t new_size)
+{
+ return realloc(ptr, new_size);
+}
+
+void my_free(void *ctx, void *ptr)
+{
+ free(ptr);
+}
+
+void setup_custom_allocators(void)
+{
+ PyMemAllocators alloc;
+ alloc.ctx = &magic;
+ alloc.malloc = my_malloc;
+ alloc.realloc = my_realloc;
+ alloc.free = my_free;
+
+ PyMem_SetRawAllocators(&alloc);
+ PyMem_SetAllocators(&alloc);
+ PyObject_SetAllocators(&areana);
+ PyMem_SetupDebugHooks();
+}
+
--
Repository URL: http://hg.python.org/peps
More information about the Python-checkins
mailing list