[Pypi-checkins] r830 - trunk/appengine

[martin.von.loewis]

Author: martin.von.loewis
Date: Tue Jul 27 22:28:57 2010
New Revision: 830

Modified:
   trunk/appengine/fetch.py
Log:
Make mirroring more robust.


Modified: trunk/appengine/fetch.py
==============================================================================
--- trunk/appengine/fetch.py	(original)
+++ trunk/appengine/fetch.py	Tue Jul 27 22:28:57 2010
@@ -1,4 +1,4 @@
-import httplib, xmlrpclib, time, pickle, urllib2, binascii, os
+import httplib, xmlrpclib, time, pickle, urllib2, binascii, os, logging, re
 try:
   from xml.etree.cElementTree import *
 except ImportError:
@@ -7,6 +7,7 @@
   except ImportError:
     from elementtree.ElementTree import *
 from google.appengine.api.labs import taskqueue
+from google.appengine.api.urlfetch import DownloadError
 import model
 
 UA = 'appengine-mirror'
@@ -64,7 +65,7 @@
 
 def package(m, todo, name):
     data = simple_page(m, name)
-    if not name:
+    if not data:
         return
     x = fromstring(data)
     for a in x.findall(".//a"):
@@ -85,7 +86,14 @@
     if f:
         h.putheader("If-none-match", f.etag)
     h.endheaders()
-    r = h.getresponse()
+    try:
+        r = h.getresponse()
+    except DownloadError, e:
+        # move job to the end of the list
+        logging.error('Downloading %s failed: %s' % (path, str(e)))
+        j = todo[0]
+        todo.insert(-1, j)
+        return
     if r.status == 304:
         # Not modified
         return
@@ -138,5 +146,15 @@
     m.todo = pickle.dumps(todo)
     m.put()
     if todo:
-        taskqueue.add(url='/step')
+        # name the task, so that no two of them will be added
+        try:
+            n, p = todo[0]
+            if n == 'file':
+                p = p[1]
+            name = '%s-%s' % (n, p)
+            name = re.sub('[^a-zA-Z0-9-]', '-', name)
+            taskqueue.add(name=name, url='/step')
+        except taskqueue.InvalidTaskError, e:
+            # likely, task already existed, or was tombstoned
+            logging.error("Queing task failed: %s" % str(e))
     return "OK (%s %s)" % (action, param)