[C++-sig] Conversion of python files to C++ ostreams

[Christopher Bruns]

I am trying to coax boost.python to automatically convert python files
to C++ std::ostreams, for methods that take arguments of type
"std::ostream&".  I have made some progress, and I could use some
advice on how to go further.

I created a derived class of std::ostream, called FilestarOstream,
which takes a std::FILE* as it constructor argument.  Then I created a
conversion from "python file" to the FilestarOstream class.  Now my
python module can automatically convert a python file to a
FilestarOstream for any method that takes a "const FilestarOstream&"
as an argument.  Ideally I would prefer to automatically convert a
python file for any method that takes a (non-const) "std::ostream&" as
an argument.

I have included my simplified source code below

My questions:

  1 - Is there an easier way to do this?  Does boost.python have a
built in incantation for wrapping methods that take a std::ostream
reference, to take python files?

  2 - Is there a way to make the type conversion work for non-const
reference arguments?  (see print_hello_wrapper2() below)

  3 - Why does my print_hello_wrapper3() not work?  It takes a const
std::ostream reference argument.  I can pass it a FileStarOstream, so
python knows how to convert a FileStarOstream to a std::ostream.  Plus
python knows how to convert a python file to a const FileStarOstream
reference.  So why can't it convert a python file to a const ostream
reference?

  4 - In my FilestarOstream_from_pyfile::convertible method, is there
a way to check that the object is not only a file, but that it is also
writable?

  5 - The PyFile_AsFile docs say "If the caller will ever use the
returned FILE* object while the GIL is released it must also call the
PyFile_IncUseCount()  and PyFile_DecUseCount()  functions described
below as appropriate."  What is "the GIL".  If I call
PyFile_IncUseCount() when I construct the FileStarOstream, when would
I call the corresponding PyFile_DecUseCount()?

Thanks in advance for any helpful tips.

##################
//// begin test.hpp ////////
#ifndef TEST_API_PYFILE_OSTREAM_H_
#define TEST_API_PYFILE_OSTREAM_H_

#include <iostream>

//  This is the sort of method I wish to wrap
std::ostream& print_hello(std::ostream& os, int foo);

// FilestarOstream is intended to aid python wrapping of methods that
// take ostream& args by converting an opaque C++ FILE* to a C++ ostream.
// This is just one part of the (yet unproven) task of automatically
// wrapping methods that take an ostream& as an argument.
class FilestarOstream : public std::ostream
{
public:
    // std::ostreams can be constructed using an std::streambuf
    // so the first step is to construct a specialized std::streambuf
    // based on a FILE*
    // Adapted from
    //  http://mail.python.org/pipermail/cplusplus-sig/2002-June/000896.html
    class std_obuf: public std::streambuf
    {
    public:
        std_obuf(std::FILE* file): m_file(file) {}
        std::FILE* updFilestar() {return m_file;}
    protected:
        std::streambuf::int_type overflow(std::streambuf::int_type c) {
            return std::fputc(c, m_file) == EOF ?
std::streambuf::traits_type::eof() : c;
        }
        std::FILE* m_file;
    };

    FilestarOstream(std::FILE* fp)
        : buf(fp), std::ostream(&buf) {}

    // Default constructor uses stdout
    FilestarOstream()
        : buf(stdout), std::ostream(&buf) {}

    std::FILE* updFilestar() {return buf.updFilestar();}

protected:
    std_obuf buf;
};

#endif // TEST_API_PYFILE_OSTREAM_H_
//////////// end test.hpp //////////


###################
//// test.cpp ///
#include "test.hpp"

std::ostream& print_hello(std::ostream& os, int foo) {
    os << "Hello, foo = " << foo << std::endl;
    return os;
}
/// end test.cpp ///


######################
// boost.python wrapping code
#include "boost/python.hpp"
#include "test.hpp"

namespace bp = boost::python;

// Three ways of wrapping print_hello() to take a python file argument,
// only one of which works:

// print_hello_wrapper1() works, but I wish I did not need a wrapper at all
std::ostream& print_hello_wrapper1(const FilestarOstream& os, int foo) {
    FilestarOstream& os_nc = const_cast<FilestarOstream&>(os);
    return print_hello(os_nc, foo);
}
// A wrapper that takes a non-const reference does not work
// Boost.Python.ArgumentError ...
std::ostream& print_hello_wrapper2(FilestarOstream& os, int foo) {
    return print_hello(os, foo);
}
// A wrapper that takes the a const reference to the base class,
ostream, also fails
// Boost.Python.ArgumentError ...
std::ostream& print_hello_wrapper3(const std::ostream& os, int foo) {
    std::ostream& os_nc = const_cast<std::ostream&>(os);
    return print_hello(os_nc, foo);
}

// Define automatic interconversion of python file <==> FilestarOstream
struct FilestarOstream_to_pyfile // untested...
{
    static PyObject* convert(FilestarOstream& os)
    {return PyFile_FromFile(os.updFilestar(), "FilestarOstream", "w", NULL);}
};
struct FilestarOstream_from_pyfile
{
    FilestarOstream_from_pyfile() {
        bp::converter::registry::push_back(
            &convertible, &construct, bp::type_id<FilestarOstream>());
    }

    static void* convertible(PyObject* obj_ptr)
    {
        if( !PyFile_Check( obj_ptr ) ) {return 0;}
        // TODO - is there a way to check whether file is writable also?
        return obj_ptr;
    }

    static void construct(
            PyObject* obj_ptr,
            bp::converter::rvalue_from_python_stage1_data* data)
    {
        // TODO - the PyFile_AsFile docs say:
        // "If the caller will ever use the returned FILE* object
while the GIL is released it must also call the PyFile_IncUseCount()
and PyFile_DecUseCount()  functions described below as appropriate."
        std::FILE* file = PyFile_AsFile(obj_ptr);
        typedef
bp::converter::rvalue_from_python_storage<FilestarOstream>
filestarOstream_storage;
        void* const storage =
reinterpret_cast<filestarOstream_storage*>(data)->storage.bytes;
        new (storage) FilestarOstream(file);
        data->convertible = storage;
    }
};

BOOST_PYTHON_MODULE(test_mod){
    // wrap std::ostream, so python will know how to convert
FilestarOstream to ostream
    bp::class_< std::ostream, boost::noncopyable >( "std_ostream",
bp::no_init );

    { //::FilestarOstream
        typedef bp::class_< FilestarOstream, bp::bases< std::ostream
>, boost::noncopyable > FilestarOstream_exposer_t;
        FilestarOstream_exposer_t FilestarOstream_exposer =
FilestarOstream_exposer_t( "FilestarOstream", bp::init< std::FILE *
>(( bp::arg("fp") )) );
        bp::scope FilestarOstream_scope( FilestarOstream_exposer );
        FilestarOstream_exposer.def( bp::init< >() );
    }

    // wrap print_hello, using one of those wrappers defined above
    bp::def(
        "print_hello",
        // choose your wrapper
          // &print_hello, // no wrapper at all, Boost.Python.ArgumentError
          &print_hello_wrapper1, // OK
          // &print_hello_wrapper2, // non-const arg =>
Boost.Python.ArgumentError
          // &print_hello_wrapper3, // base class arg =>
Boost.Python.ArgumentError
        ( bp::arg("os"), bp::arg("foo") ),
        bp::return_internal_reference<2>());

    // Register conversion from python file to FilestarOstream
    bp::to_python_converter<FilestarOstream, FilestarOstream_to_pyfile>;
    FilestarOstream_from_pyfile();
}
// end boost.python wrapping code

##### python test program ######
import test_mod
import sys

# First test using explicit FilestarOstream
# succeeds with all wrappers,
# including unwrapped print_hello
stream1 = test_mod.FilestarOstream()
test_mod.print_hello(stream1, 1)

# Remaining tests that use python files
# only work with wrappers that take an
# argument of const FilestarOstream&
stream2 = sys.stdout
test_mod.print_hello(stream2, 2)

stream3 = open('test.txt', "w")
test_mod.print_hello(stream3, 3)
stream3.close()
##### end test program ####