Parsing stream of JSON objects incrementally

[Miki Tebeka]

You probably need to accumulate a buffer and try to decode it, when succeeded return the object and read more. See example below (note that for sockets select might be a better option for reading data).

import json
from time import sleep

def json_decoder(fo):
    buff = ''
    decode = json.JSONDecoder().raw_decode
    while True:
        line = fo.readline()
        if not line:
            break
        buff += line
        print('BUFF: {}'.format(buff))
        try:
            obj, i = decode(buff)
            buff = buff[i:].lstrip()
            yield obj
        except ValueError as e:
            print('ERR: {}'.format(e))
            sleep(0.01) # select will probably be a better option :)


def main():
    import sys
    for obj in json_decoder(sys.stdin):
        print(obj)

if __name__ == '__main__':
    main()