[Twisted-Python] Uploading multiple files using ftpclient in Twisted

9 Jul 2010

      Dear twisted-python expert,

I am build engineer and I need to upload binary to ftp server.
Binary files are about 30 and each file size is about 30M.

The current script is uploading using ftplib and it takes time about 1 hour.
I want to change this script to use twisted asynchronous function.
I thought if I use asynchronous function in twisted like following,
then file uploading will be executed in parallel.
But this was executed sequentially. Uploading second file starts afer
completing first file upload.
Could you check what was wrong in my source code? Or Am I wrong in
understanding asynchronous function?

        for file in fileList:
    	ftpClient.cwd("/tmp").addCallbacks(uploadFiles, fail,
callbackArgs=(file,ftpClient))

Thanks,
Jaepyoung

Full source code:

def filesucess(ab):
    print ab

def uploadFiles(result, file,ftpClient):
    d1,d2=ftpClient.storeFile('/tmp/'+file)
    d1.addCallback(cbStore, file).addErrback(fileTransferFail)
    d1.addCallback(filesucess)
    return d2

def fileTransferFail(failure):
    failure.printTraceback()
    reactor.stop()

def showBuffer(result, bufferProtocol):
    print 'Got data:'
    print bufferProtocol.buffer.getvalue()

class Options(usage.Options):
    optParameters = [['host', 'h', 'localhost'],
                     ['port', 'p', 21],
                     ['username', 'u', 'user'],
                     ['password', None, 'password'],
                     ['passive', None, 0],
                     ['debug', 'd', 1],
                    ]
def cbStore(consumer, filename):
    fs = FileSender()
    print filename+" cbstror"
    d = fs.beginFileTransfer(open(filename, 'r'), consumer)
    d.addCallback(lambda _: consumer.finish()).addErrback(fileTransferFail)
    return d

def run():
    # Get config
    config = Options()
    config.parseOptions()
    config.opts['port'] = int(config.opts['port'])
    config.opts['passive'] = int(config.opts['passive'])
    config.opts['debug'] = int(config.opts['debug'])

    # Create the client
    FTPClient.debug = config.opts['debug']
    creator = ClientCreator(reactor, FTPClient, config.opts['username'],
                            config.opts['password'],
passive=config.opts['passive'])
    print config.opts['password']
    creator.connectTCP(config.opts['host'],
config.opts['port'],timeout=10).addCallback(connectionMade).addErrback(connectionFailed)
    reactor.run()

def connectionFailed(f):
    print "Connection Failed:", f
    reactor.stop()

def connectionMade(ftpClient):
    # Get the current working directory
    ftpClient.pwd().addCallbacks(success, fail)
    fileList = os.listdir('./temp')
    for file in fileList:
    	ftpClient.cwd("/tmp").addCallbacks(uploadFiles, fail,
callbackArgs=(file,ftpClient))
    print "connectionmade"

[Twisted-Python] Uploading multiple files using ftpclient in Twisted

Jaepyoung Kim