Hello,

I am wrapping a C library that uses an asynchronous programming model.

Operations in the C library are invoked by calling a function and including as one of the arguments a callback function that is to be called when the operation is completed. The function call to start the operation returns immediately and the work is done by a separate thread(s). When the operation is complete the separate thread(s) calls the callback function provided as input.

Below is a skeleton code sample that demonstrates how I am attempting to wrap the C library.

I would be grateful if the experts could verify that I am using PyGILState_Ensure()/PyGILState_Release() correctly, or if not please help me understand what I am doing wrong.

Also, is the call to PyEval_InitThreads() before the C library creates any new threads necessary and sufficient?

Some Python code that I wrote that uses my wrapping module (and a large number of other modules) is occasionally experiencing deadlock. I am attempting to debug that and would like to rule out errors in how I have wrapped the C library.

I appreciate any input you have.

Also, right now my need is to support using this wrapper module with Python 2.4.x and Python 2.5.x.

#include <stdio.h> #include <string.h> #include <unistd.h> #include <ctype.h>

#include "mylibrary.h"

#include "Python.h"

// used to store pointers to the Python objects that should // be used during a callback for completion of operation 01 typedef struct { PyObject * pyfunction; // Python object for the Python function to call as callback PyObject * pyarg; // Python object for the Python argument to pass in to the callback } operation01_callback_bucket_t;

// callback for the completion of operation 01 static void operation01_callback(void * user_data) { PyObject * func; PyObject * arglist; PyObject * result; PyObject * arg;

operation01_callback_bucket_t  * callbackBucket;

// cast the user_data that the libraries are passing in to the
// callback structure where we previously stored the Python function and
// arguments to call
callbackBucket = (operation01_callback_bucket_t *) user_data;

// we need to obtain the Python GIL before this thread can manipulate any Python object
PyGILState_STATE gstate;
gstate = PyGILState_Ensure();

// pick off the function and argument pointers we want to pass back into Python
func = callbackBucket -> pyfunction;
arg = callbackBucket -> pyarg;

// prepare the arg list to pass into the Python callback function
arglist = Py_BuildValue("(O)", arg);

// now call the Python callback function
result = PyEval_CallObject(func, arglist);

if (result == NULL) {
    // something went wrong so print to stderr
    PyErr_Print();
}

// take care of reference handling
Py_DECREF(arglist);
Py_XDECREF(result);

// release the Python GIL from this thread
PyGILState_Release(gstate);

// free the space the callback bucket was holding
free(callbackBucket);

return;

}

// initialize operation 01 PyObject * operation01(PyObject *self, PyObject *args) { PyObject * completeCallbackFunctionObj; PyObject * completeCallbackArgObj;

operation01_callback_bucket_t * callbackBucket = NULL;

int result;

// get Python arguments
if (!PyArg_ParseTuple(args, "OO", 
        &completeCallbackFunctionObj,
        &completeCallbackArgObj
        )){
    PyErr_SetString(PyExc_RuntimeError, "unable to parse arguments");
    return NULL;
}

// create a third party callback struct to hold the callback information
callbackBucket = (operation01_callback_bucket_t *) malloc(sizeof(operation01_callback_bucket_t));
callbackBucket -> pyfunction = completeCallbackFunctionObj;
callbackBucket -> pyarg = completeCallbackArgObj;

// since we are holding pointers to these objects we need to increase
// the reference count for each
Py_XINCREF(callbackBucket -> pyfunction);
Py_XINCREF(callbackBucket -> pyarg);

// kick off operation01

Py_BEGIN_ALLOW_THREADS

result = operation01(
                    operation01_complete_callback,
                    (void *) callbackBucket
                    );

Py_END_ALLOW_THREADS

if (result != 0){
    sprintf(msg, "rc = %d: unable to start operation01", result);
    PyErr_SetString(PyExc_RuntimeError, msg);
    return NULL;
}

// return None to indicate success
Py_RETURN_NONE;

}

// method table mapping names to wrappers static PyMethodDef mywrappermethods[] = { {"operation01", operation01, METH_VARARGS}, {NULL, NULL} };

// module initialization function void initmywrapper(){ PyObject * module; PyObject * moduleDict;

// this is called before the C library creates any threads
PyEval_InitThreads();

// get handle to the module dictionary
module = Py_InitModule("mywrapper", mywrappermethods);
moduleDict = PyModule_GetDict(module);

}