[Twisted-web] Agent and request timeout

3 Jul 2011

      I have some progress on the request timeout.

I have something that seems to work. But I am sure I have got something
wrong..

This is a copy of the response.py example with some additions to deal with
request timeouts.

Any chance that someone could take a look and let me know where I have gone
wrong.

#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from pprint import pformat

from twisted.internet import reactor
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Protocol
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
import urllib
import time

class BeginningPrinter(Protocol):
    def __init__(self, finished):
        self.finished = finished
        self.remaining = 1024 * 10

    def dataReceived(self, bytes):
        if self.remaining:
            display = bytes[:self.remaining]
            print 'Some data received:', bytes, 'no bytes', len(bytes)
            print display
            self.remaining -= len(display)

    def connectionLost(self, reason):
        print 'Finished receiving body:', reason.getErrorMessage()
        self.finished.callback(None)

def do_cancel(d):
    print 'got do cancel'
    d._canceller(d)

from stringprod_resp import StringProducer
my_data = ['fred','ted','red','fred']
for c, a in enumerate(my_data):
    if c > 2:
        time.sleep(10)
    agent = Agent(reactor)
    #postdata = urllib.urlencode({'Username' : 'nsp05682',
    #                             'Password' : 'rocs345',
    #                             'Action' : 'CheckCredits'})
    postdata = urllib.urlencode({'the-field' : a})
    body = StringProducer(postdata)
    print 'body', body
    d = agent.request(
        'POST',
        'http://www.johna.com/test',
        Headers({'User-Agent': ['Twisted Web Client'],
                 'Accept-Encoding':['identity'],
                 'Content-Type':['application/x-www-form-urlencoded']}),
        body)
    #             'Content-Length':[str(len(postdata))],

    def cbRequest(response):
        print 'Response version:', response.version
        print 'Response code:', response.code
        print 'Response phrase:', response.phrase
        print 'Response headers:'
        print pformat(list(response.headers.getAllRawHeaders()))
        finished = Deferred()
        print 'response deliverbody',response.deliverBody
        response.deliverBody(BeginningPrinter(finished))
        return finished
    d.addCallback(cbRequest)
    #reactor.callLater(10, check_timeout,d)
    #j = dir(d)
    #print 'later', j
    #print 'fff cancel', d.cancel
    #print 'fff canceller', d._canceller
    #k = dir(agent.request)
    #print 'kkk', k
    #print 'reacotcallater'
    #if a == 'fred':
    #    print 'cancel ', a
    no_response = reactor.callLater(40, do_cancel, d)
        #d.cancel()

    def got_error(o):
        print 'got error', str(o)
        reason = dir(o)
        print 'got reason', o.getErrorMessage

    def cbShutdown(ignored, no_response):
        print 'got finish ignored', ignored
        #print 'no rsposne', no_response
        norep = dir(no_response)
        #print 'kkk', norep
        if no_response.cancelled:
            print 'already cancelled'
        else:
            print 'not cancelled'
            no_response.cancel()
            no_response = None
        #reactor.stop()
    d.addCallback(cbShutdown, no_response)
    d.addErrback(got_error)

reactor.run()

[Twisted-web] Agent and request timeout

John Aherne