ipc pickle

[Gerson Kurz]

Here is a suggestion:
------------------------------------------------
import cPickle
import StringIO
import struct
import os

# your classes
class Client:
    def __init__(self, file):
        self.file = file
    def read(self):
        # read length and load object
        x = os.read(self.file, 4)
        length = struct.unpack('i', x)[0]
        buf = os.read(self.file, length)
        f = StringIO.StringIO(buf)
        return cPickle.load(f)
        
class Server:
    def __init__(self, file):
        self.file = file

    def write(self, obj):
        # write length of serialozed object, write serialized object
        f = StringIO.StringIO()
        cPickle.dump(obj, f, 1)
        val = f.getvalue()
        length = len(val)
        plen = struct.pack("i", length)
        os.write(self.file, plen)
        os.write(self.file,val)

# my classes        
class Client2:
    def __init__(self, file):
        self.file = file
    def read(self):
        # assume size repr is less than 20 bytes
        x = os.read(self.file, 20)
        pos = x.find('.')
        size_expected = int(x[:pos])
        if size_expected > 20:
            # read rest of data
            more_data = os.read(self.file, size_expected - pos )
            return cPickle.loads(x[pos+1:] + more_data)
        else:
            return cPickle.loads(x[pos+1:])
        
class Server2:
    def __init__(self, file):
        self.file = file

    def write(self, obj):
        test = cPickle.dumps(obj)
        # interestingly, this seems to be slower:
        #os.write(self.file,"%d." % len(test) + test)
        os.write(self.file,"%d.%s" % (len(test),test))

# test script
if __name__=='__main__':
    rp, wp = os.pipe()

    import time

    test_string = u"this is a test string that is more than 20 bytes
long"

    c = Client(rp)
    s = Server(wp)
    start = time.clock()
    for i in xrange(3000):
        s.write(test_string)
        assert(test_string==c.read())
    print time.clock() - start

    c = Client2(rp)
    s = Server2(wp)
    start = time.clock()
    for i in xrange(3000):
        s.write(test_string)
        assert(test_string==c.read())
    print time.clock() - start


# results:
0.518367439587 (your)
0.228796436055 (mine)
-------------------------------------------------------

But, your version seems reasonably fast already :). My "Full-Blown
RPCServer" that uses dedicated threads for function calls (so that one
RPC call can issue another RPC call while not blocking other threads
making their RPC calls etc) is *a lot* slower...