[Pypi-checkins] r777 - in trunk/pypi: . tools
martin.von.loewis
python-checkins at python.org
Thu Jul 22 23:47:37 CEST 2010
Author: martin.von.loewis
Date: Thu Jul 22 23:47:37 2010
New Revision: 777
Added:
trunk/pypi/tools/demodata
trunk/pypi/tools/mksqlite (contents, props changed)
Modified:
trunk/pypi/README
trunk/pypi/admin.py
trunk/pypi/config.ini.template
trunk/pypi/config.py
trunk/pypi/pkgbase_schema.sql
trunk/pypi/store.py
trunk/pypi/webui.py
Log:
Add support for running PyPI on sqlite3.
Modified: trunk/pypi/README
==============================================================================
--- trunk/pypi/README (original)
+++ trunk/pypi/README Thu Jul 22 23:47:37 2010
@@ -10,7 +10,7 @@
- zope.tal
- zope.tales
- zope.i18nmessageid
-- psycopg2
+- psycopg2 (for testing, sqlite3 might be sufficient)
- docutils
- distutils2
@@ -32,7 +32,7 @@
$ virtualenv --no-site-packages --distribute .
$ bin/easy_install cElementTree zope.interface zope.pagetemplate
$ bin/easy_install zope.tal zope.tales zope.i18nmessageid psycopg2
- $ bin/easy_install M2Crypto BeautifulSoup docutils
+ $ bin/easy_install docutils
Then you can launch the server using the pypi.wsgi script::
@@ -41,3 +41,12 @@
PyPI will be available in your browser at http://localhost:8000
+Database Setup
+--------------
+
+To fill a database, run pkgbase_schema.sql on an empty Postgres database.
+Then run tools/demodata to populate the database with dummy data.
+
+For testing purposes, run tools/mksqlite to create packages.db. Set
+[database]driver to sqlite3, and [database]name to packages.db, then
+run tools/demodata to populate the database.
\ No newline at end of file
Modified: trunk/pypi/admin.py
==============================================================================
--- trunk/pypi/admin.py (original)
+++ trunk/pypi/admin.py Thu Jul 22 23:47:37 2010
@@ -41,27 +41,30 @@
raise ValueError, "user is not currently owner"
store.delete_role(owner, 'Owner', package)
-def add_classifier(store, classifier):
+def add_classifier(st, classifier):
''' Add a classifier to the trove_classifiers list
'''
- cursor = store.get_cursor()
+ cursor = st.get_cursor()
cursor.execute("select max(id) from trove_classifiers")
- id = int(cursor.fetchone()[0]) + 1
+ id = cursor.fetchone()[0]
+ if id:
+ id = int(id) + 1
+ else:
+ id = 1
fields = [f.strip() for f in classifier.split('::')]
for f in fields:
assert ':' not in f
levels = []
for l in range(2, len(fields)):
c2 = ' :: '.join(fields[:l])
- cursor.execute('select id from trove_classifiers where classifier=%s', (c2,))
+ store.safe_execute(cursor, 'select id from trove_classifiers where classifier=%s', (c2,))
l = cursor.fetchone()
if not l:
raise ValueError, c2 + " is not a known classifier"
levels.append(l[0])
levels += [id] + [0]*(3-len(levels))
- cursor.execute('insert into trove_classifiers (id, classifier, l2, l3, l4, l5) '
+ store.safe_execute(cursor, 'insert into trove_classifiers (id, classifier, l2, l3, l4, l5) '
'values (%s,%s,%s,%s,%s,%s)', [id, classifier]+levels)
- print 'done'
def rename_package(store, old, new):
''' Rename a package. '''
@@ -135,6 +138,7 @@
remove_package(*args)
elif command == 'addclass':
add_classifier(*args)
+ print 'done'
elif command == 'addowner':
add_owner(*args)
elif command == 'delowner':
Modified: trunk/pypi/config.ini.template
==============================================================================
--- trunk/pypi/config.ini.template (original)
+++ trunk/pypi/config.ini.template Thu Jul 22 23:47:37 2010
@@ -1,4 +1,5 @@
[database]
+driver = postgresql2
name = packages
user = pypi
files_dir = /MacDev/svn.python.org/pypi-pep345/files
@@ -8,7 +9,7 @@
mailhost = mail.commonground.com.au
adminemail = richard at commonground.com.au
replyto = richard at commonground.com.au
-url = http://localhost/cgi-bin/pypi.cgi
+url = http://localhost:8000/pypi
pydotorg = http://www.python.org/
simple_script = /simple
Modified: trunk/pypi/config.py
==============================================================================
--- trunk/pypi/config.py (original)
+++ trunk/pypi/config.py Thu Jul 22 23:47:37 2010
@@ -8,6 +8,10 @@
c.read(configfile)
self.database_name = c.get('database', 'name')
self.database_user = c.get('database', 'user')
+ if c.has_option('database', 'driver'):
+ self.database_driver = c.get('database', 'driver')
+ else:
+ self.database_driver = 'psycopg2'
if c.has_option('database', 'password'):
self.database_pw = c.get('database', 'password')
else:
Modified: trunk/pypi/pkgbase_schema.sql
==============================================================================
--- trunk/pypi/pkgbase_schema.sql (original)
+++ trunk/pypi/pkgbase_schema.sql Thu Jul 22 23:47:37 2010
@@ -72,9 +72,11 @@
);
CREATE INDEX journals_name_idx ON journals(name);
CREATE INDEX journals_version_idx ON journals(version);
+-- nosqlite
CREATE INDEX journals_latest_releases ON
journals(submitted_date, name, version)
WHERE version IS NOT NULL AND action='new release';
+-- nosqlite-end
CREATE INDEX journals_changelog ON
journals(submitted_date, name, version, action);
@@ -145,10 +147,12 @@
-- trove ids sequence
+-- nosqlite
CREATE TABLE dual (dummy INTEGER);
INSERT INTO dual VALUES (1);
CREATE SEQUENCE trove_ids;
SELECT setval('trove_ids', 1000) FROM dual;
+-- nosqlite-end
-- Table structure for table: release_classifiers
@@ -312,9 +316,9 @@
name TEXT PRIMARY KEY,
value TIMESTAMP
);
-INSERT INTO timestamps(name, value) VALUES('http','1970-01-01');
-INSERT INTO timestamps(name, value) VALUES('ftp','1970-01-01');
-INSERT INTO timestamps(name, value) VALUES('browse_tally','1970-01-01');
+INSERT INTO timestamps(name, value) VALUES('http','1970-01-01 00:00:00');
+INSERT INTO timestamps(name, value) VALUES('ftp','1970-01-01 00:00:00');
+INSERT INTO timestamps(name, value) VALUES('browse_tally','1970-01-01 00:00:00');
-- Table structure for table: timestamps
-- Note: stamp_name is ftp, http
Modified: trunk/pypi/store.py
==============================================================================
--- trunk/pypi/store.py (original)
+++ trunk/pypi/store.py Thu Jul 22 23:47:37 2010
@@ -1,7 +1,16 @@
''' Implements a store of disutils PKG-INFO entries, keyed off name, version.
'''
-import sys, os, re, psycopg2, time, hashlib, random, types, math, stat, errno
+import sys, os, re, time, hashlib, random, types, math, stat, errno
import logging, cStringIO, string, datetime, calendar, binascii, urllib2, cgi
+try:
+ import psycopg2
+except ImportError:
+ pass
+try:
+ import sqlite3
+ sqlite3_cursor = sqlite3.Cursor
+except ImportError:
+ sqlite3_cursor = type(None)
from xml.parsers import expat
from distutils.version import LooseVersion
import trove, openid2rp
@@ -156,6 +165,9 @@
if params is None:
return cursor.execute(sql)
+ if isinstance(cursor, sqlite3_cursor):
+ sql = sql.replace('%s', "?")
+
# Encode every incoming param to UTF-8 if it's a string
safe_params = []
for param in params:
@@ -181,6 +193,12 @@
self._conn = None
self._cursor = None
self._trove = None
+ if self.config.database_driver == 'sqlite3':
+ self.true, self.false = '1', '0'
+ self.can_lock = False
+ else:
+ self.true, self.false = 'TRUE', 'FALSE'
+ self.can_lock = True
def trove(self):
if not self._trove:
@@ -339,7 +357,7 @@
# hide all other releases of this package if thus configured
if self.get_package_autohide(name):
safe_execute(cursor, 'update releases set _pypi_hidden=%s where '
- 'name=%s and version <> %s', ('TRUE', name, version))
+ 'name=%s and version <> %s', (self.true, name, version))
# add description urls
if html:
@@ -594,8 +612,8 @@
where = ' %s '%operator.join(where)
if '_pypi_hidden' in spec:
- if spec['_pypi_hidden'] in ('1', 1): v = 'TRUE'
- else: v = 'FALSE'
+ if spec['_pypi_hidden'] in ('1', 1): v = self.true
+ else: v = self.false
if where:
where += ' AND _pypi_hidden = %s'%v
else:
@@ -762,7 +780,7 @@
where j.version is not NULL
and j.action = 'new release'
and j.name = r.name and j.version = r.version
- and r._pypi_hidden = FALSE
+ and r._pypi_hidden = '''+self.false+'''
and j.submitted_date > %s
order by submitted_date desc
''', (time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),))
@@ -833,7 +851,7 @@
from journals j, releases r
where j.version is not NULL
and j.name = r.name and j.version = r.version
- and r._pypi_hidden = FALSE
+ and r._pypi_hidden = '''+self.false+'''
order by submitted_date desc
''')
@@ -1026,13 +1044,13 @@
'''Add a user rating of a release; message is optional'''
cursor = self.get_cursor()
safe_execute(cursor, '''insert into ratings (name, version, user_name, date, rating)
- values(%s, %s, %s, now(), %s)''', (name, version, self.username, rating))
+ values(%s, %s, %s, current_timestamp, %s)''', (name, version, self.username, rating))
if message:
safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to)
- values(currval('ratings_id_seq'), %s, now(), %s, NULL)''',
+ values(currval('ratings_id_seq'), %s, current_timestamp, %s, NULL)''',
(self.username, message))
safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action)
- values(%s,%s,currval('comments_id_seq'),%s,now(),%s)''',
+ values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''',
(name, version, self.username, 'add_rating %r' % message))
def copy_rating(self, name, fromversion, toversion):
@@ -1040,7 +1058,7 @@
return the comment if any'''
cursor = self.get_cursor()
safe_execute(cursor, '''insert into ratings(name,version,user_name,date,rating)
- select name,%s,user_name,now(),rating from ratings
+ select name,%s,user_name,current_timestamp,rating from ratings
where name=%s and version=%s and user_name=%s''',
(toversion, name, fromversion, self.username))
# only copy comment, not follow-ups
@@ -1052,7 +1070,7 @@
select currval('ratings_id_seq'), user_name, date, message, in_reply_to
from comments where id=%s''', (cid,))
safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action)
- values(%s, %s, currval('comments_id_seq'), %s, now(), %s)''', (name, toversion,
+ values(%s, %s, currval('comments_id_seq'), %s, current_timestamp, %s)''', (name, toversion,
self.username, 'copied %s' % cid))
safe_execute(cursor, '''select message from comments
@@ -1064,7 +1082,7 @@
'''Remove a rating for the current user'''
cursor = self.get_cursor()
safe_execute(cursor, """insert into comments_journal(name, version, id, submitted_by, date, action)
- select %s, %s, id, %s, now(), 'deleted' from ratings where user_name=%s and name=%s and version=%s""",
+ select %s, %s, id, %s, current_timestamp, 'deleted' from ratings where user_name=%s and name=%s and version=%s""",
(name, version, self.username, self.username, name, version))
safe_execute(cursor, "delete from ratings where user_name=%s and name=%s and version=%s",
(self.username, name, version))
@@ -1099,9 +1117,9 @@
safe_execute(cursor, "select c.rating, r.name, r.version from comments c, ratings r where c.id=%s and c.rating=r.id", (msg,))
rating, name, version = cursor.fetchone()
safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to)
- values(%s,%s,now(),%s,%s)''', (rating, self.username, comment, msg))
+ values(%s,%s,current_timestamp,%s,%s)''', (rating, self.username, comment, msg))
safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action)
- values(%s,%s,currval('comments_id_seq'),%s,now(),%s)''', (name, version, self.username,
+ values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', (name, version, self.username,
'add %s %r' % (msg, comment)))
return name, version
@@ -1112,7 +1130,7 @@
name, version = cursor.fetchone()
safe_execute(cursor, "delete from comments where id=%s", (msg,))
safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action)
- values(%s, %s, %s, %s, now(), 'delete')''', (name, version, msg, self.username))
+ values(%s, %s, %s, %s, current_timestamp, 'delete')''', (name, version, msg, self.username))
def has_package_comments(self, name):
"Return true if the package has any comments"
@@ -1283,7 +1301,7 @@
(name, ))
return int(cursor.fetchone()[0])
- def store_user(self, name, password, email, gpg_keyid, otk=True):
+ def store_user(self, name, password, email, gpg_keyid="", otk=True):
''' Store info about the user to the database.
The "password" argument is passed in cleartext and sha-ed
@@ -1323,7 +1341,7 @@
if not otk:
return None
otk = ''.join([random.choice(chars) for x in range(32)])
- safe_execute(cursor, 'insert into rego_otk (name, otk, date) values (%s, %s, now())',
+ safe_execute(cursor, 'insert into rego_otk (name, otk, date) values (%s, %s, current_timestamp)',
(name, otk))
return otk
@@ -1508,7 +1526,8 @@
# Regenerate tally. First, release locks we hold on the timestamps
self._conn.commit()
# Clear old tally
- cursor.execute("lock table browse_tally")
+ if self.can_lock:
+ cursor.execute("lock table browse_tally")
cursor.execute("delete from browse_tally")
# Regenerate tally; see browse() below
cursor.execute("""insert into browse_tally
@@ -1516,7 +1535,7 @@
from trove_classifiers t, release_classifiers rc, releases r
where rc.name=r.name and rc.version=r.version and not r._pypi_hidden and rc.trove_id=t.id
group by t.l2, rc.name, rc.version) res group by res.l2""")
- cursor.execute("update timestamps set value=now() where name='browse_tally'")
+ cursor.execute("update timestamps set value=current_timestamp where name='browse_tally'")
self._conn.commit()
cursor.execute("select trove_id, tally from browse_tally")
return [], cursor.fetchall()
@@ -1534,7 +1553,7 @@
return [], cursor.fetchall()
# First compute statement to produce all packages still selected
- pkgs = "select name, version, summary from releases where _pypi_hidden=FALSE"
+ pkgs = "select name, version, summary from releases where _pypi_hidden="+self.false
for c in selected_classifiers:
level = t.trove[c].level
pkgs = """select distinct a.name, a.version, summary from (%s) a, release_classifiers rc, trove_classifiers t
@@ -1586,7 +1605,7 @@
cursor = self.get_cursor()
sql = '''insert into release_files (name, version, python_version,
packagetype, comment_text, filename, md5_digest, upload_time) values
- (%s, %s, %s, %s, %s, %s, %s, now())'''
+ (%s, %s, %s, %s, %s, %s, %s, current_timestamp)'''
safe_execute(cursor, sql, (name, version, pyversion, filetype,
comment, filename, md5_digest))
@@ -1764,9 +1783,9 @@
name, last_seen = users[0]
if datetime.datetime.now()-datetime.timedelta(0,60) > last_seen:
# refresh cookie and login time every minute
- sql = 'update cookies set last_seen=now() where cookie=%s'
+ sql = 'update cookies set last_seen=current_timestamp where cookie=%s'
safe_execute(cursor, sql, (cookie,))
- sql ='update users set last_login=now() where name=%s'
+ sql ='update users set last_login=current_timestamp where name=%s'
safe_execute(cursor, sql, (name,))
return name
return None
@@ -1776,7 +1795,7 @@
cursor = self.get_cursor()
cookie = binascii.hexlify(os.urandom(16))
sql = '''insert into cookies(cookie, name, last_seen)
- values(%s, %s, now())'''
+ values(%s, %s, current_timestamp)'''
safe_execute(cursor, sql, (cookie, username))
return cookie
@@ -1790,7 +1809,7 @@
cursor = self.get_cursor()
# Check for existing session
sql = '''select id,url, assoc_handle from openid_sessions
- where provider=%s and expires>now()'''
+ where provider=%s and expires>current_timestamp'''
safe_execute(cursor, sql, (provider[0],))
sessions = cursor.fetchall()
if sessions:
@@ -1827,7 +1846,7 @@
cursor = self.get_cursor()
# Check for existing session
sql = '''select assoc_handle from openid_sessions
- where provider=%s and url=%s and expires>now()'''
+ where provider=%s and url=%s and expires>current_timestamp'''
safe_execute(cursor, sql, (claimed, endpoint,))
sessions = cursor.fetchall()
if sessions:
@@ -1915,6 +1934,9 @@
# already closed
connection = None
return self.open()
+ elif self.config.database_driver == 'sqlite3':
+ self._conn = connection = sqlite3.connect(self.config.database_name,
+ detect_types=sqlite3.PARSE_DECLTYPES)
else:
self._conn = connection = psycopg2.connect(**cd)
@@ -1938,8 +1960,8 @@
if self.has_user(username):
self.username = username
if update_last_login:
- self.get_cursor().execute('''
- update users set last_login=now() where name=%s''', (username,))
+ safe_execute(self.get_cursor(), '''
+ update users set last_login=current_timestamp where name=%s''', (username,))
self.userip = userip
def setpasswd(self, username, password):
Added: trunk/pypi/tools/demodata
==============================================================================
--- (empty file)
+++ trunk/pypi/tools/demodata Thu Jul 22 23:47:37 2010
@@ -0,0 +1,54 @@
+#!/usr/bin/python
+import sys, os, urllib
+
+root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(root)
+import admin, store, config
+
+cfg = config.Config(root+'/config.ini')
+st = store.Store(cfg)
+
+# classifiers
+for c in urllib.urlopen("http://pypi.python.org/pypi?%3Aaction=list_classifiers").read().splitlines():
+ admin.add_classifier(st, c)
+
+# Demo data starts here
+
+# an admin
+otk = st.store_user('fred', 'fredpw', 'fred at python.test')
+st.delete_otk(otk)
+st.add_role('fred', 'Admin', None)
+# an owner
+otk = st.store_user('barney', 'barneypw', 'barney at python.test')
+st.delete_otk(otk)
+
+# package spam
+st.set_user('barney', '127.0.0.1', True)
+for version in ('0.8', '0.9', '1.0'):
+ st.store_package('spam', version, {
+ 'author':'Barney Geroellheimer',
+ 'author_email':'barney at python.test',
+ 'homepage':'http://spam.python.test/',
+ 'license':'GPL',
+ 'summary':'The spam package',
+ 'description':'Does anybody want to provide real data here?',
+ 'classifiers':["Development Status :: 6 - Mature",
+ "Programming Language :: Python :: 2"],
+ '_pypi_hidden':False
+ })
+
+# package eggs
+for version in ('0.1', '0.2', '0.3', '0.4'):
+ st.store_package('eggs', version, {
+ 'author':'Barney Geroellheimer',
+ 'author_email':'barney at python.test',
+ 'homepage':'http://eggs.python.test/',
+ 'license':'GPL',
+ 'summary':'The eggs package',
+ 'description':'Does anybody want to provide real data here?',
+ 'classifiers':["Development Status :: 3 - Alpha",
+ "Programming Language :: Python :: 3"],
+ '_pypi_hidden':version!='0.4'
+ })
+
+st.commit()
Added: trunk/pypi/tools/mksqlite
==============================================================================
--- (empty file)
+++ trunk/pypi/tools/mksqlite Thu Jul 22 23:47:37 2010
@@ -0,0 +1,26 @@
+#!/usr/bin/python
+import os
+dbpath = "packages.db"
+
+if os.path.exists(dbpath):
+ print "Remove",dbpath,"first"
+ raise SystemExit
+
+print "Creating database", dbpath
+sqlite = os.popen('sqlite3 '+dbpath, "w")
+passthrough = True
+for line in open('pkgbase_schema.sql'):
+ if 'nosqlite-end' in line:
+ # end of disabled block
+ passthrough = True
+ print >>sqlite
+ continue
+ if 'nosqlite' in line:
+ passthrough = False
+ print >>sqlite
+ continue
+ if not passthrough:
+ print >> sqlite
+ continue
+ sqlite.write(line)
+sqlite.close()
Modified: trunk/pypi/webui.py
==============================================================================
--- trunk/pypi/webui.py (original)
+++ trunk/pypi/webui.py Thu Jul 22 23:47:37 2010
@@ -6,7 +6,12 @@
from distutils.util import rfc822_escape
from distutils2.metadata import DistributionMetadata
-import psycopg2
+try:
+ import psycopg2
+ OperationalError = psycopg2.OperationalError
+except ImportError:
+ class OperationalError(Exception):
+ pass
try:
import cElementTree
@@ -259,6 +264,7 @@
self.store = store.Store(self.config)
try:
try:
+ self.store.get_cursor() # make sure we can connect
self.inner_run()
except NotFound:
self.fail('Not Found', code=404)
@@ -288,7 +294,7 @@
except IOError, error:
# ignore broken pipe errors (client vanished on us)
if error.errno != 32: raise
- except psycopg2.OperationalError, message:
+ except OperationalError, message:
# clean things up
self.store.force_close()
message = str(message)
More information about the Pypi-checkins
mailing list