I want to get the contents of a list of URLs, and if the getPage is successful then I want to write it to a file. If it fails, I want to log a message. In addition, I would like to have this process run every five minutes. The problem is how to specify a yes/no scenario when the callbacks take arguments. Also, is the best way to loop to just put FeederProtocol.start() in a loop? any ehlp would be greatly appreciated. while true: get page if error: log else: write to file sleep 5 minutes class FeederProtocol(object): def get_page_error(self, _failure, log, feed_name): """ Failure while getting page. """ msg = 'While getting %s: %s' % (feed_name, _failure.getTraceback()) log.error(msg) _failure.trap(RuntimeError) # Caused chain to stop raise(RuntimeError) get_page(url): return client.getPage(url, timeout=30) def page_to_file(self, feed, name): try: fnout = os.path.join(app_conf['download_dir'], name) fout = open(fnout, 'w') except IOError, e: log.error('Opening %s: %s' % (fnout, e)) return try: fout.write(feed) except IOError, e: log.error('Wrting to %s: %s' % (fnout, e)) finally: fout.close() def start(self, urls): for url in urls: d = defer.succeed(self.printStatus()) d.addCallback(self.get_page, feed, log, app_conf) d.addCallbacks((self.page_to_file, log, app_conf), (self.get_page_error, log, feed[0])) class FeederFactory(protocol.ClientFactory): protocol = FeederProtocol() def __init__(self, url_list): urls = url_list self.start(self.urls) def start(self, urls): protocol.start(urls) if __name__=="__main__": name = 'mytest' urls = ['www.example.com', 'www.example2.com'] f = FeederFactory(urls) reactor.run()
participants (2)
-
Aaron DeVore
-
hancock.robert@gmail.com