[Numpy-discussion] cPickle.loads and Numeric

[Robert Kern]

On Tue, Feb 25, 2014 at 8:19 AM, Chris <chris.madan at gmail.com> wrote:
> I have some old code that uses cPickle.loads which used to work, but now
> reports an error in loading the module Numeric. Since Numeric has been
> replaced by numpy, this makes sense, but, how can I get cPickle.loads to
> work? I tested the code again on an older machine and it works fine
> there, but, I'd like to get it working again on a modern set-up as well.

It's relatively straightforward to subclass Unpickler to redirect it
when it goes to look for the array constructor that it expects from
the Numeric module.


from cStringIO import StringIO
import pickle

import numpy as np


TEST_NUMERIC_PICKLE = ('\x80\x02cNumeric\narray_constructor\nq\x01(K\x05\x85U'
                       '\x01lU(\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00'
                       '\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x03'
                       '\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00'
                       '\x00\x00K\x01tRq\x02.')


# Constant from Numeric.
LittleEndian = 1

def array_constructor(shape, typecode, thestr, Endian=LittleEndian):
    """ The old Numeric array constructor for pickle, recast for numpy.
    """
    if typecode == "O":
        x = np.array(thestr, "O")
    else:
        x = np.fromstring(thestr, typecode)
    x.shape = shape
    if LittleEndian != Endian:
        return x.byteswapped()
    else:
        return x


class NumericUnpickler(pickle.Unpickler):
    """ Allow loading of pickles containing Numeric arrays and
    converting them to numpy arrays.
    """

    def find_class(self, module, name):
        """ Return the constructor callable for a given "class".

        Overridden to handle Numeric.array_constructor specially.
        """
        if module == 'Numeric' and name == 'array_constructor':
            return array_constructor
        else:
            return pickle.Unpickler.find_class(self, module, name)


def load(fp):
    return NumericUnpickler(fp).load()


def loads(pickle_string):
    fp = StringIO(pickle_string)
    return NumericUnpickler(fp).load()


if __name__ == '__main__':
    import sys
    print loads(TEST_NUMERIC_PICKLE)
    # Look, Ma! No Numeric!
    assert 'Numeric' not in sys.modules

-- 
Robert Kern