wrap the queue to multiprocess download

alias 1248283536 at qq.com
Thu Sep 8 10:12:19 EDT 2011


here is my part of program,you can see main structure,i want to wrap it in class ,
class   webdata(object):
    def  __init__(self,arg):
        jobs = Queue.Queue()
        for  x  in  name:
            jobs.put(self.url)
                
    
          
    def download(self):
        while not self.jobs.empty():
            url = self.jobs.get()
            hx = httplib2.Http()
            resp, content = hx.request(url, headers=headers).read()
            self.jobs.task_done()


    def  myrun(self):
        for i in range(30):
              threading.Thread(target=self.download).start()
        self.jobs.join()
    
if  __name__=="__main__":

    s=webdata('quote')
    s.myrun()

when it run ,the output is  :
Traceback (most recent call last):
  File "/usr/lib/python2.7/threading.py", line 552, in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/threading.py", line 505, in run
    self.__target(*self.__args, **self.__kwargs)
  File "/home/pengtao/workspace/try.py", line 75, in download
    resp, content = hx.request(url, headers=headers).read()
  File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 1288, in request
    (scheme, authority, request_uri, defrag_uri) = urlnorm(uri)
  File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 201, in urlnorm
    (scheme, authority, path, query, fragment) = parse_uri(uri)
  File "/usr/local/lib/python2.7/dist-packages/httplib2/__init__.py", line 197, in parse_uri
    groups = URI.match(uri).groups()
TypeError: expected string or buffer


when i use the same  structrue, don't   wrap it ,it can run .
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.python.org/pipermail/python-list/attachments/20110908/6e09cdbf/attachment.html>


More information about the Python-list mailing list