[Numpy-discussion] tracing numpy data allocation with python callbacks

Thouis Jones thouis.jones at curie.fr
Wed May 16 15:34:02 EDT 2012


I recently had need of tracing numpy data allocation/deallocation.  I
was unable to find a simple way to do so, and so ended up putting the
code below into ndarraytypes.h to allow me to trace allocations.  A
key part is that this jumps back into python, so I can inspect the
stack and find out where in the upper level code large allocations are
occurring.

I wondered, however, if there were a better way to accomplish the same
goal, preferably in pure python.

Ray Jones


static void *numpy_memtrace_malloc(size_t size) {
  void *result = malloc(size);
  PyObject *npy = PyImport_ImportModule("numpy");
  if (npy != NULL) {
    if (PyObject_HasAttrString(npy, "tracemalloc")) {
      PyObject *ret;
      ret = PyObject_CallMethod(npy, "tracemalloc", "Nl",
PyLong_FromVoidPtr(result), (long) size);
      Py_XDECREF(ret);
      Py_DECREF(npy);
      PyErr_Clear();
    }
  }
  return result;
}

static void numpy_memtrace_free(void *ptr) {
  free(ptr);
  PyObject *npy = PyImport_ImportModule("numpy");
  if (npy != NULL) {
    if (PyObject_HasAttrString(npy, "tracefree")) {
      PyObject *ret;
      ret = PyObject_CallMethod(npy, "tracefree", "N", PyLong_FromVoidPtr(ptr));
      Py_XDECREF(ret);
      Py_DECREF(npy);
      PyErr_Clear();
    }
  }
}

static void *numpy_memtrace_realloc(void *ptr, size_t size) {
  void *result = realloc(ptr, size);
  PyObject *npy = PyImport_ImportModule("numpy");
  if (npy != NULL) {
    if (PyObject_HasAttrString(npy, "npyealloc")) {
      PyObject *ret;
      ret = PyObject_CallMethod(npy, "npyealloc", "NNl",
PyLong_FromVoidPtr(ptr), PyLong_FromVoidPtr(result), size);
      Py_XDECREF(ret);
      Py_DECREF(npy);
      PyErr_Clear();
    }
  }
  return result;
}

  /* Data buffer */
#define PyDataMem_NEW(size) ((char *)numpy_memtrace_malloc(size))
#define PyDataMem_FREE(ptr)  numpy_memtrace_free(ptr)
#define PyDataMem_RENEW(ptr,size) ((char *)numpy_memtrace_realloc(ptr,size))



More information about the NumPy-Discussion mailing list