[Pypi-checkins] r827 - in trunk/appengine: . templates
martin.von.loewis
python-checkins at python.org
Sun Jul 25 22:55:26 CEST 2010
Author: martin.von.loewis
Date: Sun Jul 25 22:55:25 2010
New Revision: 827
Added:
trunk/appengine/fetch.py (contents, props changed)
trunk/appengine/templates/dir.html (contents, props changed)
Modified:
trunk/appengine/handlers.py
trunk/appengine/mirror.py
trunk/appengine/model.py
trunk/appengine/templates/index.html
Log:
Some mirroring implemented.
Added: trunk/appengine/fetch.py
==============================================================================
--- (empty file)
+++ trunk/appengine/fetch.py Sun Jul 25 22:55:25 2010
@@ -0,0 +1,142 @@
+import httplib, xmlrpclib, time, pickle, urllib2, binascii, os
+try:
+ from xml.etree.cElementTree import *
+except ImportError:
+ try:
+ from xml.etree.ElementTree import *
+ except ImportError:
+ from elementtree.ElementTree import *
+from google.appengine.api.labs import taskqueue
+import model
+
+UA = 'appengine-mirror'
+
+def rpc():
+ return xmlrpclib.ServerProxy('http://pypi.python.org/pypi')
+
+def simple_page(m, project):
+ h = httplib.HTTPConnection('pypi.python.org')
+ if project:
+ h.putrequest('GET', '/simple/'+urllib2.quote(project)+'/')
+ obj = model.Project.get_by_key_name(project)
+ else:
+ h.putrequest('GET', '/simple/')
+ obj = m
+ h.putheader('User-Agent', UA)
+ h.endheaders()
+ r = h.getresponse()
+ html = r.read()
+ if r.status == 404:
+ return None
+ if r.status == 301:
+ # package not existant anymore, however, similarly-spelled
+ # package exists
+ return None
+ if r.status != 200:
+ raise ValueError, "Status %d on %s" % (r.status, project)
+ if obj:
+ obj.simple = html
+ else:
+ obj = model.Project(key_name=project, simple=html)
+ if project:
+ # the root index is not signed
+ h.putrequest('GET', '/serversig/'+urllib2.quote(project)+'/')
+ h.putheader('User-Agent', UA)
+ h.endheaders()
+ r = h.getresponse()
+ obj.sig = r.read()
+ obj.put()
+ return html
+
+def get_state():
+ t = time.time()
+ m = model.MirrorState.all().fetch(1)
+ if m:
+ return m[0]
+ m = model.MirrorState()
+ todo = [('package', '')]
+ for p in rpc().list_packages():
+ todo.append(('package', p))
+ todo.append(('packages_listed', t))
+ m.todo = pickle.dumps(todo)
+ m.put()
+ return m
+
+def package(m, todo, name):
+ data = simple_page(m, name)
+ if not name:
+ return
+ x = fromstring(data)
+ for a in x.findall(".//a"):
+ url = a.attrib['href']
+ if not url.startswith('../../packages/'):
+ continue
+ url = url.split('#')[0]
+ url = url[len('../../packages/'):]
+ # insert after current task
+ todo.insert(1,('file', (name, url)))
+
+def copy_file(m, todo, (package, path)):
+ f = model.File.get_by_key_name(path.split('/'))
+ if f:
+ f = f[0]
+ h = httplib.HTTPConnection('pypi.python.org')
+ h.putrequest('HEAD', '/packages/'+path)
+ if f:
+ h.putheader("If-none-match", f.etag)
+ h.endheaders()
+ r = h.getresponse()
+ if r.status == 304:
+ # Not modified
+ return
+ if r.status == 200:
+ if not f:
+ #import pdb, sys
+ #sys.__stdout__.write('\a')
+ #sys.__stdout__.flush()
+ #debugger = pdb.Pdb(stdin=sys.__stdin__, stdout=sys.__stdout__)
+ #debugger.set_trace(sys._getframe().f_back)
+ parent, name = path.rsplit("/",1)
+ parent = model.Directory.mkdir(parent)
+ secret = binascii.b2a_hex(os.urandom(10))
+ f = model.File(parent=parent, key_name=name, path=path,
+ etag=r.msg['etag'], secret=secret)
+ else:
+ f.contents.delete()
+ f.etag = r.msg['etag']
+ f.put()
+ else:
+ raise ValueError, "Bad HTTP status"
+
+
+def packages_listed(m, todo, t):
+ # reappend after all files
+ todo.append('last_modified', t)
+
+def last_modified(m, todo, t):
+ m.last_modified = t
+
+
+actions = {'package':package,
+ 'file':copy_file,
+ 'packages_listed': packages_listed,
+ 'last_modified': last_modified,
+ }
+
+def step():
+ m = get_state()
+ todo = pickle.loads(m.todo)
+ if not m.todo:
+ return
+ action, param = todo[0]
+ try:
+ actions[action](m, todo, param)
+ except Exception, e:
+ raise
+ return str(e)
+ del todo[0]
+ m.todo = pickle.dumps(todo)
+ m.put()
+ if todo and 0:
+ taskqueue.add(url='/step')
+ return "OK (%s %s)" % (action, param)
Modified: trunk/appengine/handlers.py
==============================================================================
--- trunk/appengine/handlers.py (original)
+++ trunk/appengine/handlers.py Sun Jul 25 22:55:25 2010
@@ -1,26 +1,62 @@
-from google.appengine.ext import webapp
-from google.appengine.ext.webapp import template
-import model
+import os
+from google.appengine.ext import webapp, blobstore
+from google.appengine.ext.webapp import template, blobstore_handlers
+import model, fetch
def tpl_path(template_file_name):
return os.path.join(os.path.dirname(__file__), 'templates', template_file_name)
class Index(webapp.RequestHandler):
def get(self):
- self.response.out.write(template.render(tpl_path('index.html', {})))
+ self.response.out.write(template.render(tpl_path('index.html'), {}))
class Simple(webapp.RequestHandler):
def get(self, path):
path = path.rstrip('/')
- p = model.Project.all().filter(name=path)
+ if not path:
+ p = model.MirrorState.all().fetch(1)[0]
+ else:
+ p = model.Project.get_by_key_name(path)
self.response.out.write(p.simple)
-class File(webapp.RequestHandler):
- def get(self):
- #package
- pass
+class Upload(blobstore_handlers.BlobstoreUploadHandler):
+ def post(self):
+ path = self.request.get('path')
+ # File object should have been created upfront
+ f = model.File.all().filter('path = ', path).fetch()[0]
+ f.contents = self.get_uploads('file')[0]
+ f.put()
+ fetch.received(f)
+ self.response.set_status(204)
+
+class File(blobstore_handlers.BlobstoreDownloadHandler):
+ def get(self, path):
+ if path:
+ path = path.rstrip("/")
+ f = model.File.all().filter("path = ", path).fetch(1)
+ else:
+ f = None
+ if f:
+ if f[0].contents:
+ self.send_blob(f.contents)
+ else:
+ self.response.headers['content-type'] = 'text/plain'
+ self.response.error(404)
+ self.response.out.write('Not yet copied')
+ return
+ if path:
+ d = model.Directory.all().filter("path = ", path)
+ if not d:
+ self.response.error(404)
+ return
+ else:
+ d = None
+ dirs = model.Directory.all().filter('parent = ', d).fetch(100)
+ files = model.File.all().filter('parent = ', d).fetch(1000)
+ self.response.out.write(template.render(tpl_path('dir.html'), {
+ 'path':path, 'dirs':dirs, 'files':files }))
-class ServerSig(webapp.RequestHandler):
+class Serversig(webapp.RequestHandler):
def get(self):
#serversig
pass
@@ -30,5 +66,8 @@
#localstats/days
pass
-
-
+class Step(webapp.RequestHandler):
+ def get(self):
+ self.response.headers['content-type'] = 'text/plain'
+ self.response.out.write(fetch.step())
+ post = get
Modified: trunk/appengine/mirror.py
==============================================================================
--- trunk/appengine/mirror.py (original)
+++ trunk/appengine/mirror.py Sun Jul 25 22:55:25 2010
@@ -5,9 +5,10 @@
application = webapp.WSGIApplication(
[('/', Index),
('/simple/(.*)', Simple),
- ('/package/(.*)', File),
+ ('/packages/(.*)', File),
('/serversig/(.*)', Serversig),
('/local-stats/days/(.*)', Stats),
+ ('/step', Step)],
debug=True)
def main():
Modified: trunk/appengine/model.py
==============================================================================
--- trunk/appengine/model.py (original)
+++ trunk/appengine/model.py Sun Jul 25 22:55:25 2010
@@ -1,16 +1,44 @@
-from google.appengine.ext import db
+from google.appengine.ext import db, blobstore
class MirrorState(db.Model):
# singleton object
todo = db.BlobProperty()
+ simple = db.BlobProperty() # simple root
last_modified = db.StringProperty()
class Project(db.Model):
- name = db.StringProperty()
simple = db.BlobProperty()
+ sig = db.BlobProperty()
+
+class Directory(db.Model):
+ path = db.StringProperty()
+ @staticmethod
+ def mkdir(path):
+ if '/' not in path:
+ res = Directory.get_by_key_name(path, parent=None)
+ if not res:
+ res = Directory(parent=None, key_name=path, path=path)
+ res.put()
+ return res
+ parent, name = path.rsplit('/', 1)
+ parent = Directory.mkdir(parent)
+ res = Directory.get_by_key_name(name, parent=parent)
+ if not res:
+ res = Directory(parent=parent, key_name=name, path=path)
+ res.put()
+ return res
+
+ @property
+ def name(self):
+ return self.key().name()
+
class File(db.Model):
- contents = db.BlobProperty()
+ # also using parent and key
+ contents = blobstore.BlobReferenceProperty()
etag = db.StringProperty()
project = db.ReferenceProperty(Project)
path = db.StringProperty()
+ @property
+ def name(self):
+ return self.key().name()
Added: trunk/appengine/templates/dir.html
==============================================================================
--- (empty file)
+++ trunk/appengine/templates/dir.html Sun Jul 25 22:55:25 2010
@@ -0,0 +1,13 @@
+<html>
+<head>
+<title>{{path}}</title>
+</head>
+<body>
+{% for dir in dirs %}
+<a href="/packages/{{path}}{{dir.name}}/">{{dir}}</a><br/>
+{% endfor %}
+{% for file in files %}
+<a href="/packages/{{path}}{{file}}">{{file}}</a><br/>
+{% endfor %}
+</body>
+</html>
\ No newline at end of file
Modified: trunk/appengine/templates/index.html
==============================================================================
--- trunk/appengine/templates/index.html (original)
+++ trunk/appengine/templates/index.html Sun Jul 25 22:55:25 2010
@@ -4,6 +4,7 @@
</head>
<body>
<a href="simple/">simple<a><br>
+<a href="packages/">packages<a><br>
<a href="serversig/">serversig<a><br>
<a href="local-stats/days/">stats<a><br>
</body> </html>
More information about the Pypi-checkins
mailing list