[Python-checkins] CVS: python/dist/src/Include mymalloc.h,2.17,2.18 objimpl.h,2.14,2.15

Guido van Rossum python-dev@python.org
Wed, 3 May 2000 19:44:56 -0400 (EDT)


Update of /projects/cvsroot/python/dist/src/Include
In directory eric:/projects/python/develop/guido/clean/Include

Modified Files:
	mymalloc.h objimpl.h 
Log Message:
Vladimir Marangozov's long-awaited malloc restructuring.
For more comments, read the patches@python.org archives.
For documentation read the comments in mymalloc.h and objimpl.h.

(This is not exactly what Vladimir posted to the patches list; I've
made a few changes, and Vladimir sent me a fix in private email for a
problem that only occurs in debug mode.  I'm also holding back on his
change to main.c, which seems unnecessary to me.)



Index: mymalloc.h
===================================================================
RCS file: /projects/cvsroot/python/dist/src/Include/mymalloc.h,v
retrieving revision 2.17
retrieving revision 2.18
diff -C2 -r2.17 -r2.18
*** mymalloc.h	1998/12/04 18:48:10	2.17
--- mymalloc.h	2000/05/03 23:44:23	2.18
***************
*** 58,61 ****
--- 58,63 ----
  #endif
  
+ #include "myproto.h"
+ 
  #ifdef __cplusplus
  /* Move this down here since some C++ #include's don't like to be included
***************
*** 68,77 ****
  #endif
  
! /* The following should never be necessary */
! #ifdef NEED_TO_DECLARE_MALLOC_AND_FRIEND
! extern ANY *malloc Py_PROTO((size_t));
! extern ANY *calloc Py_PROTO((size_t, size_t));
! extern ANY *realloc Py_PROTO((ANY *, size_t));
! extern void free Py_PROTO((ANY *)); /* XXX sometimes int on Unix old systems */
  #endif
  
--- 70,75 ----
  #endif
  
! #ifndef DL_IMPORT       /* declarations for DLL import */
! #define DL_IMPORT(RTYPE) RTYPE
  #endif
  
***************
*** 88,118 ****
  #endif
  
! #define PyMem_NEW(type, n) \
! 	( (type *) malloc(_PyMem_EXTRA + (n) * sizeof(type)) )
! #define PyMem_RESIZE(p, type, n) \
! 	if ((p) == NULL) \
! 		(p) =  (type *) malloc(_PyMem_EXTRA + (n) * sizeof(type)); \
! 	else \
! 		(p) = (type *) realloc((ANY *)(p), \
! 				       _PyMem_EXTRA + (n) * sizeof(type))
! #define PyMem_DEL(p) free((ANY *)p)
! #define PyMem_XDEL(p) if ((p) == NULL) ; else PyMem_DEL(p)
  
  
! /* Two sets of function wrappers around malloc and friends; useful if
!    you need to be sure that you are using the same memory allocator as
!    Python.  Note that the wrappers make sure that allocating 0 bytes
!    returns a non-NULL pointer, even if the underlying malloc doesn't.
!    The Python interpreter continues to use PyMem_NEW etc. */
  
! /* These wrappers around malloc call PyErr_NoMemory() on failure */
! extern DL_IMPORT(ANY *) Py_Malloc Py_PROTO((size_t));
! extern DL_IMPORT(ANY *) Py_Realloc Py_PROTO((ANY *, size_t));
! extern DL_IMPORT(void) Py_Free Py_PROTO((ANY *));
  
! /* These wrappers around malloc *don't* call anything on failure */
  extern DL_IMPORT(ANY *) PyMem_Malloc Py_PROTO((size_t));
  extern DL_IMPORT(ANY *) PyMem_Realloc Py_PROTO((ANY *, size_t));
  extern DL_IMPORT(void) PyMem_Free Py_PROTO((ANY *));
  
  #ifdef __cplusplus
--- 86,199 ----
  #endif
  
! /*
!  * Core memory allocator
!  * =====================
!  */
! 
! /* To make sure the interpreter is user-malloc friendly, all memory
!    APIs are implemented on top of this one.
! 
!    The PyCore_* macros can be defined to make the interpreter use a
!    custom allocator. Note that they are for internal use only. Both
!    the core and extension modules should use the PyMem_* API. */
! 
! #ifndef PyCore_MALLOC_FUNC
! #undef PyCore_REALLOC_FUNC
! #undef PyCore_FREE_FUNC
! #define PyCore_MALLOC_FUNC      malloc
! #define PyCore_REALLOC_FUNC     realloc
! #define PyCore_FREE_FUNC        free
! #endif
  
+ #ifndef PyCore_MALLOC_PROTO
+ #undef PyCore_REALLOC_PROTO
+ #undef PyCore_FREE_PROTO
+ #define PyCore_MALLOC_PROTO     Py_PROTO((size_t))
+ #define PyCore_REALLOC_PROTO    Py_PROTO((ANY *, size_t))
+ #define PyCore_FREE_PROTO       Py_PROTO((ANY *))
+ #endif
  
! #ifdef NEED_TO_DECLARE_MALLOC_AND_FRIEND
! extern ANY *PyCore_MALLOC_FUNC PyCore_MALLOC_PROTO;
! extern ANY *PyCore_REALLOC_FUNC PyCore_REALLOC_PROTO;
! extern void PyCore_FREE_FUNC PyCore_FREE_PROTO;
! #endif
  
! #ifndef PyCore_MALLOC
! #undef PyCore_REALLOC
! #undef PyCore_FREE
! #define PyCore_MALLOC(n)        PyCore_MALLOC_FUNC(n)
! #define PyCore_REALLOC(p, n)    PyCore_REALLOC_FUNC((p), (n))
! #define PyCore_FREE(p)          PyCore_FREE_FUNC(p)
! #endif
  
! /* BEWARE:
! 
!    Each interface exports both functions and macros. Extension modules
!    should normally use the functions for ensuring binary compatibility
!    of the user's code across Python versions. Subsequently, if the
!    Python runtime switches to its own malloc (different from standard
!    malloc), no recompilation is required for the extensions.
! 
!    The macro versions trade compatibility for speed. They can be used
!    whenever there is a performance problem, but their use implies
!    recompilation of the code for each new Python release. The Python
!    core uses the macros because it *is* compiled on every upgrade.
!    This might not be the case with 3rd party extensions in a custom
!    setup (for example, a customer does not always have access to the
!    source of 3rd party deliverables). You have been warned! */
! 
! /*
!  * Raw memory interface
!  * ====================
!  */
! 
! /* Functions */
! 
! /* Function wrappers around PyCore_MALLOC and friends; useful if you
!    need to be sure that you are using the same memory allocator as
!    Python.  Note that the wrappers make sure that allocating 0 bytes
!    returns a non-NULL pointer, even if the underlying malloc
!    doesn't. Returned pointers must be checked for NULL explicitly.
!    No action is performed on failure. */
  extern DL_IMPORT(ANY *) PyMem_Malloc Py_PROTO((size_t));
  extern DL_IMPORT(ANY *) PyMem_Realloc Py_PROTO((ANY *, size_t));
  extern DL_IMPORT(void) PyMem_Free Py_PROTO((ANY *));
+ 
+ /* Starting from Python 1.6, the wrappers Py_{Malloc,Realloc,Free} are
+    no longer supported. They used to call PyErr_NoMemory() on failure. */
+ 
+ /* Macros */
+ #define PyMem_MALLOC(n)         PyCore_MALLOC(n)
+ #define PyMem_REALLOC(p, n)     PyCore_REALLOC((ANY *)(p), (n))
+ #define PyMem_FREE(p)           PyCore_FREE((ANY *)(p))
+ 
+ /*
+  * Type-oriented memory interface
+  * ==============================
+  */
+ 
+ /* Functions */
+ #define PyMem_New(type, n) \
+ 	( (type *) PyMem_Malloc((n) * sizeof(type)) )
+ #define PyMem_Resize(p, type, n) \
+ 	( (p) = (type *) PyMem_Realloc((n) * sizeof(type)) )
+ #define PyMem_Del(p) PyMem_Free(p)
+ 
+ /* Macros */
+ #define PyMem_NEW(type, n) \
+ 	( (type *) PyMem_MALLOC(_PyMem_EXTRA + (n) * sizeof(type)) )
+ #define PyMem_RESIZE(p, type, n) \
+ 	if ((p) == NULL) \
+ 		(p) = (type *)(PyMem_MALLOC( \
+ 				    _PyMem_EXTRA + (n) * sizeof(type))); \
+ 	else \
+ 		(p) = (type *)(PyMem_REALLOC((p), \
+ 				    _PyMem_EXTRA + (n) * sizeof(type)))
+ #define PyMem_DEL(p) PyMem_FREE(p)
+ 
+ /* PyMem_XDEL is deprecated. To avoid the call when p is NULL,
+    it is recommended to write the test explicitly in the code.
+    Note that according to ANSI C, free(NULL) has no effect. */
  
  #ifdef __cplusplus

Index: objimpl.h
===================================================================
RCS file: /projects/cvsroot/python/dist/src/Include/objimpl.h,v
retrieving revision 2.14
retrieving revision 2.15
diff -C2 -r2.14 -r2.15
*** objimpl.h	2000/03/01 15:06:53	2.14
--- objimpl.h	2000/05/03 23:44:23	2.15
***************
*** 36,75 ****
  ******************************************************************/
  
  /*
! Additional macros for modules that implement new object types.
  You must first include "object.h".
  
! PyObject_NEW(type, typeobj) allocates memory for a new object of the given
! type; here 'type' must be the C structure type used to represent the
! object and 'typeobj' the address of the corresponding type object.
! Reference count and type pointer are filled in; the rest of the bytes of
! the object are *undefined*!  The resulting expression type is 'type *'.
! The size of the object is actually determined by the tp_basicsize field
! of the type object.
! 
! PyObject_NEW_VAR(type, typeobj, n) is similar but allocates a variable-size
! object with n extra items.  The size is computed as tp_basicsize plus
! n * tp_itemsize.  This fills in the ob_size field as well.
! */
  
! #ifndef MS_COREDLL
  extern DL_IMPORT(PyObject *) _PyObject_New Py_PROTO((PyTypeObject *));
  extern DL_IMPORT(PyVarObject *) _PyObject_NewVar Py_PROTO((PyTypeObject *, int));
! 
! #define PyObject_NEW(type, typeobj) ((type *) _PyObject_New(typeobj))
! #define PyObject_NEW_VAR(type, typeobj, n) ((type *) _PyObject_NewVar(typeobj, n))
! 
! #else
! /* For an MS-Windows DLL, we change the way an object is created, so that the
!    extension module's malloc is used, rather than the core DLL malloc, as there is
!    no guarantee they will use the same heap
! */
! extern DL_IMPORT(PyObject *) _PyObject_New Py_PROTO((PyTypeObject *, PyObject *));
! extern DL_IMPORT(PyVarObject *) _PyObject_NewVar Py_PROTO((PyTypeObject *, int, PyVarObject *));
! 
! #define PyObject_NEW(type, typeobj) ((type *) _PyObject_New(typeobj,(PyObject *)malloc((typeobj)->tp_basicsize)))
! #define PyObject_NEW_VAR(type, typeobj, n) ((type *) _PyObject_NewVar(typeobj, n, (PyVarObject *)malloc((typeobj)->tp_basicsize + n * (typeobj)->tp_itemsize)))
  
! #endif /* MS_COREDLL */
  
  #ifdef __cplusplus
--- 36,237 ----
  ******************************************************************/
  
+ #include "mymalloc.h"
+ 
  /*
! Functions and macros for modules that implement new object types.
  You must first include "object.h".
+ 
+  - PyObject_New(type, typeobj) allocates memory for a new object of
+    the given type; here 'type' must be the C structure type used to
+    represent the object and 'typeobj' the address of the corresponding
+    type object.  Reference count and type pointer are filled in; the
+    rest of the bytes of the object are *undefined*!  The resulting
+    expression type is 'type *'.  The size of the object is actually
+    determined by the tp_basicsize field of the type object.
+ 
+  - PyObject_NewVar(type, typeobj, n) is similar but allocates a
+    variable-size object with n extra items.  The size is computed as
+    tp_basicsize plus n * tp_itemsize.  This fills in the ob_size field
+    as well.
+ 
+  - PyObject_Del(op) releases the memory allocated for an object.
+ 
+  - PyObject_Init(op, typeobj) and PyObject_InitVar(op, typeobj, n) are
+    similar to PyObject_{New, NewVar} except that they don't allocate
+    the memory needed for an object. Instead of the 'type' parameter,
+    they accept the pointer of a new object (allocated by an arbitrary
+    allocator) and initialize its object header fields.
+ 
+ Note that objects created with PyObject_{New, NewVar} are allocated
+ within the Python heap by an object allocator, the latter being
+ implemented (by default) on top of the Python raw memory
+ allocator. This ensures that Python keeps control on the user's
+ objects regarding their memory management; for instance, they may be
+ subject to automatic garbage collection.
+ 
+ In case a specific form of memory management is needed, implying that
+ the objects would not reside in the Python heap (for example standard
+ malloc heap(s) are mandatory, use of shared memory, C++ local storage
+ or operator new), you must first allocate the object with your custom
+ allocator, then pass its pointer to PyObject_{Init, InitVar} for
+ filling in its Python-specific fields: reference count, type pointer,
+ possibly others. You should be aware that Python has very limited
+ control over these objects because they don't cooperate with the
+ Python memory manager. Such objects may not be eligible for automatic
+ garbage collection and you have to make sure that they are released
+ accordingly whenever their destructor gets called (cf. the specific
+ form of memory management you're using).
+ 
+ Unless you have specific memory management requirements, it is
+ recommended to use PyObject_{New, NewVar, Del}. */
+ 
+ /* 
+  * Core object memory allocator
+  * ============================
+  */
+ 
+ /* The purpose of the object allocator is to make make the distinction
+    between "object memory" and the rest within the Python heap.
+    
+    Object memory is the one allocated by PyObject_{New, NewVar}, i.e.
+    the one that holds the object's representation defined by its C
+    type structure, *excluding* any object-specific memory buffers that
+    might be referenced by the structure (for type structures that have
+    pointer fields).  By default, the object memory allocator is
+    implemented on top of the raw memory allocator.
+ 
+    The PyCore_* macros can be defined to make the interpreter use a
+    custom object memory allocator. They are reserved for internal
+    memory management purposes exclusively. Both the core and extension
+    modules should use the PyObject_* API. */
+ 
+ #ifndef PyCore_OBJECT_MALLOC_FUNC
+ #undef PyCore_OBJECT_REALLOC_FUNC
+ #undef PyCore_OBJECT_FREE_FUNC
+ #define PyCore_OBJECT_MALLOC_FUNC    PyCore_MALLOC_FUNC
+ #define PyCore_OBJECT_REALLOC_FUNC   PyCore_REALLOC_FUNC
+ #define PyCore_OBJECT_FREE_FUNC      PyCore_FREE_FUNC
+ #endif
+ 
+ #ifndef PyCore_OBJECT_MALLOC_PROTO
+ #undef PyCore_OBJECT_REALLOC_PROTO
+ #undef PyCore_OBJECT_FREE_PROTO
+ #define PyCore_OBJECT_MALLOC_PROTO   PyCore_MALLOC_PROTO
+ #define PyCore_OBJECT_REALLOC_PROTO  PyCore_REALLOC_PROTO
+ #define PyCore_OBJECT_FREE_PROTO     PyCore_FREE_PROTO
+ #endif
+ 
+ #ifdef NEED_TO_DECLARE_OBJECT_MALLOC_AND_FRIEND
+ extern ANY *PyCore_OBJECT_MALLOC_FUNC PyCore_OBJECT_MALLOC_PROTO;
+ extern ANY *PyCore_OBJECT_REALLOC_FUNC PyCore_OBJECT_REALLOC_PROTO;
+ extern void PyCore_OBJECT_FREE_FUNC PyCore_OBJECT_FREE_PROTO;
+ #endif
+ 
+ #ifndef PyCore_OBJECT_MALLOC
+ #undef PyCore_OBJECT_REALLOC
+ #undef PyCore_OBJECT_FREE
+ #define PyCore_OBJECT_MALLOC(n)      PyCore_OBJECT_MALLOC_FUNC(n)
+ #define PyCore_OBJECT_REALLOC(p, n)  PyCore_OBJECT_REALLOC_FUNC((p), (n))
+ #define PyCore_OBJECT_FREE(p)        PyCore_OBJECT_FREE_FUNC(p)
+ #endif
  
! /*
!  * Raw object memory interface
!  * ===========================
!  */
! 
! /* The use of this API should be avoided, unless a builtin object
!    constructor inlines PyObject_{New, NewVar}, either because the
!    latter functions cannot allocate the exact amount of needed memory,
!    either for speed. This situation is exceptional, but occurs for
!    some object constructors (PyBuffer_New, PyList_New...).  Inlining
!    PyObject_{New, NewVar} for objects that are supposed to belong to
!    the Python heap is discouraged. If you really have to, make sure
!    the object is initialized with PyObject_{Init, InitVar}. Do *not*
!    inline PyObject_{Init, InitVar} for user-extension types or you
!    might seriously interfere with Python's memory management. */
! 
! /* Functions */
! 
! /* Wrappers around PyCore_OBJECT_MALLOC and friends; useful if you
!    need to be sure that you are using the same object memory allocator
!    as Python. These wrappers *do not* make sure that allocating 0
!    bytes returns a non-NULL pointer. Returned pointers must be checked
!    for NULL explicitly; no action is performed on failure. */
! extern DL_IMPORT(ANY *) PyObject_Malloc Py_PROTO((size_t));
! extern DL_IMPORT(ANY *) PyObject_Realloc Py_PROTO((ANY *, size_t));
! extern DL_IMPORT(void) PyObject_Free Py_PROTO((ANY *));
! 
! /* Macros */
! #define PyObject_MALLOC(n)           PyCore_OBJECT_MALLOC(n)
! #define PyObject_REALLOC(op, n)      PyCore_OBJECT_REALLOC((ANY *)(op), (n))
! #define PyObject_FREE(op)            PyCore_OBJECT_FREE((ANY *)(op))
  
! /*
!  * Generic object allocator interface
!  * ==================================
!  */
! 
! /* Functions */
! extern DL_IMPORT(PyObject *) PyObject_Init Py_PROTO((PyObject *, PyTypeObject *));
! extern DL_IMPORT(PyVarObject *) PyObject_InitVar Py_PROTO((PyVarObject *, PyTypeObject *, int));
  extern DL_IMPORT(PyObject *) _PyObject_New Py_PROTO((PyTypeObject *));
  extern DL_IMPORT(PyVarObject *) _PyObject_NewVar Py_PROTO((PyTypeObject *, int));
! extern DL_IMPORT(void) _PyObject_Del Py_PROTO((PyObject *));
  
! #define PyObject_New(type, typeobj) \
! 		( (type *) _PyObject_New(typeobj) )
! #define PyObject_NewVar(type, typeobj, n) \
! 		( (type *) _PyObject_NewVar((typeobj), (n)) )
! #define PyObject_Del(op) _PyObject_Del((PyObject *)(op))
! 
! /* Macros trading binary compatibility for speed. See also mymalloc.h.
!    Note that these macros expect non-NULL object pointers.*/
! #define PyObject_INIT(op, typeobj) \
! 	( (op)->ob_type = (typeobj), _Py_NewReference((PyObject *)(op)), (op) )
! #define PyObject_INIT_VAR(op, typeobj, size) \
! 	( (op)->ob_size = (size), PyObject_INIT((op), (typeobj)) )
! 
! #define _PyObject_SIZE(typeobj) ( (typeobj)->tp_basicsize )
! #define _PyObject_VAR_SIZE(typeobj, n) \
! 	( (typeobj)->tp_basicsize + (n) * (typeobj)->tp_itemsize )
! 
! #define PyObject_NEW(type, typeobj) \
! ( (type *) PyObject_Init( \
! 	(PyObject *) PyObject_MALLOC( _PyObject_SIZE(typeobj) ), (typeobj)) )
! #define PyObject_NEW_VAR(type, typeobj, n) \
! ( (type *) PyObject_InitVar( \
! 	(PyVarObject *) PyObject_MALLOC( _PyObject_VAR_SIZE((typeobj),(n)) ),\
! 	(typeobj), (n)) )
! #define PyObject_DEL(op) PyObject_FREE(op)
! 
! /* This example code implements an object constructor with a custom
!    allocator, where PyObject_New is inlined, and shows the important
!    distinction between two steps (at least):
!        1) the actual allocation of the object storage;
!        2) the initialization of the Python specific fields
!           in this storage with PyObject_{Init, InitVar}.
! 
!    PyObject *
!    YourObject_New(...)
!    {
!        PyObject *op;
! 
!        op = (PyObject *) Your_Allocator(_PyObject_SIZE(YourTypeStruct));
!        if (op == NULL)
!            return PyErr_NoMemory();
! 
!        op = PyObject_Init(op, &YourTypeStruct);
!        if (op == NULL)
!            return NULL;
! 
!        op->ob_field = value;
!        ...
!        return op;
!    }
! 
!    Note that in C++, the use of the new operator usually implies that
!    the 1st step is performed automatically for you, so in a C++ class
!    constructor you would start directly with PyObject_Init/InitVar. */
  
  #ifdef __cplusplus