[Pypi-checkins] r853 - in trunk/pypi: . tools
martin.von.loewis
python-checkins at python.org
Sat Aug 14 18:02:34 CEST 2010
Author: martin.von.loewis
Date: Sat Aug 14 18:02:34 2010
New Revision: 853
Added:
trunk/pypi/tools/sql-migrate-20100814.sql (contents, props changed)
Modified:
trunk/pypi/pkgbase_schema.sql
trunk/pypi/rpc.py
trunk/pypi/store.py
trunk/pypi/tools/demodata.py
Log:
Integrate dependency tables into a single one.
Modified: trunk/pypi/pkgbase_schema.sql
==============================================================================
--- trunk/pypi/pkgbase_schema.sql (original)
+++ trunk/pypi/pkgbase_schema.sql Sat Aug 14 18:02:34 2010
@@ -157,94 +157,18 @@
CREATE INDEX rel_class_trove_id_idx ON release_classifiers(trove_id);
CREATE INDEX rel_class_name_version_idx ON release_classifiers(name, version);
--- Table structure for table: release_provides
-CREATE TABLE release_provides (
+-- Release dependencies
+-- See store.py for the valid kind values
+CREATE TABLE release_dependencies (
name TEXT,
version TEXT,
+ kind INTEGER,
specifier TEXT,
FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
);
-CREATE INDEX rel_prov_name_idx ON release_provides(name);
-CREATE INDEX rel_prov_version_id_idx ON release_provides(version);
-CREATE INDEX rel_prov_name_version_idx ON release_provides (name,version);
-
-
--- Table structure for table: release_requires
-CREATE TABLE release_requires (
- name TEXT,
- version TEXT,
- specifier TEXT,
- FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
-);
-CREATE INDEX rel_req_name_idx ON release_requires(name);
-CREATE INDEX rel_req_version_id_idx ON release_requires(version);
-CREATE INDEX rel_req_name_version_idx ON release_requires(name,version);
-
--- Table structure for table: release_obsoletes
-CREATE TABLE release_obsoletes (
- name TEXT,
- version TEXT,
- specifier TEXT,
- FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
-);
-CREATE INDEX rel_obs_name_idx ON release_obsoletes(name);
-CREATE INDEX rel_obs_version_id_idx ON release_obsoletes(version);
-CREATE INDEX rel_obs_name_version_idx ON release_obsoletes (name,version);
-
--- Table structure for table: release_requires_external
-CREATE TABLE release_requires_external (
- name TEXT,
- version TEXT,
- specifier TEXT,
- FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
-);
-CREATE INDEX rel_req_ext_name_idx ON release_requires_external(name);
-CREATE INDEX rel_req_ext_version_id_idx ON release_requires_external(version);
-CREATE INDEX rel_req_ext_name_version_idx ON release_requires_external(name,version);
-
--- Table structure for table: release_requires_dist
-CREATE TABLE release_requires_dist (
- name TEXT,
- version TEXT,
- specifier TEXT,
- FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
-);
-CREATE INDEX rel_req_dist_name_idx ON release_requires_dist(name);
-CREATE INDEX rel_req_dist_version_id_idx ON release_requires_dist(version);
-CREATE INDEX rel_req_dist_name_version_idx ON release_requires_dist(name,version);
-
--- Table structure for table: release_provides_dist
-CREATE TABLE release_provides_dist (
- name TEXT,
- version TEXT,
- specifier TEXT,
- FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
-);
-CREATE INDEX rel_prov_dist_name_idx ON release_provides_dist(name);
-CREATE INDEX rel_prov_dist_version_id_idx ON release_provides_dist(version);
-CREATE INDEX rel_prov_dist_name_version_idx ON release_provides_dist(name,version);
-
--- Table structure for table: release_obsoletes_dist
-CREATE TABLE release_obsoletes_dist (
- name TEXT,
- version TEXT,
- specifier TEXT,
- FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
-);
-CREATE INDEX rel_obs_dist_name_idx ON release_obsoletes_dist(name);
-CREATE INDEX rel_obs_dist_version_id_idx ON release_obsoletes_dist(version);
-CREATE INDEX rel_obs_dist_name_version_idx ON release_obsoletes_dist(name,version);
-
--- Table structure for table: release_project_url
-CREATE TABLE release_project_url (
- name TEXT,
- version TEXT,
- specifier TEXT,
- FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
-);
-CREATE INDEX rel_proj_url_name_idx ON release_project_url(name);
-CREATE INDEX rel_proj_url_version_id_idx ON release_project_url(version);
-CREATE INDEX rel_proj_url_name_version_idx ON release_project_url(name,version);
+CREATE INDEX rel_dep_name_idx ON release_dependencies(name);
+CREATE INDEX rel_dep_name_version_idx ON release_dependencies(name, version);
+CREATE INDEX rel_dep_name_version_kind_idx ON release_dependencies(name, version, kind);
-- Table structure for table: package_files
-- python version is only first two digits
Modified: trunk/pypi/rpc.py
==============================================================================
--- trunk/pypi/rpc.py (original)
+++ trunk/pypi/rpc.py Sat Aug 14 18:02:34 2010
@@ -4,6 +4,10 @@
import time
from cStringIO import StringIO
from SimpleXMLRPCServer import SimpleXMLRPCDispatcher
+from collections import defaultdict
+
+# local imports
+from store import dependency
class RequestHandler(SimpleXMLRPCDispatcher):
"""A request dispatcher for the PyPI XML-RPC API."""
@@ -29,7 +33,8 @@
webui_obj.handler.send_header('Content-type', 'text/xml')
webui_obj.handler.send_header('charset', 'UTF-8' );
webui_obj.handler.end_headers()
- data = webui_obj.handler.rfile.read()
+ length = int(webui_obj.env['CONTENT_LENGTH'])
+ data = webui_obj.handler.rfile.read(length)
# This should be thread-safe, as the store is really a singleton
self.store = webui_obj.store
response = self._marshaled_dispatch(data)
@@ -80,11 +85,10 @@
return {}
info = info.as_dict()
del info['description_html']
- for col in ('requires', 'provides', 'obsoletes', 'requires_dist',
- 'obsoletes_dist', 'project_url', 'provides_dist',
- 'requires_external'):
- rows = store.get_release_relationships(package_name, version, col)
- info[col] = [row['specifier'] for row in rows]
+ dependencies = defaultdict(list)
+ for kind, specifier in store.get_release_dependencies(package_name, version):
+ dependencies[dependency.by_val[kind]].append(specifier)
+ info.update(dependencies)
classifiers = [r[0] for r in store.get_release_classifiers(package_name,
version)]
info['classifiers' ] = classifiers
Modified: trunk/pypi/store.py
==============================================================================
--- trunk/pypi/store.py (original)
+++ trunk/pypi/store.py Sat Aug 14 18:02:34 2010
@@ -2,6 +2,7 @@
'''
import sys, os, re, time, hashlib, random, types, math, stat, errno
import logging, cStringIO, string, datetime, calendar, binascii, urllib2, cgi
+from collections import defaultdict
try:
import psycopg2
except ImportError:
@@ -33,6 +34,24 @@
]
dist_file_types_d = dict(dist_file_types)
+# This could have been done with Postgres ENUMs, however
+# a) they are not extensible, and
+# b) they are not supported in other databases
+class dependency:
+ requires = 1
+ provides = 2
+ obsoletes = 3
+ requires_dist = 4
+ provides_dist = 5
+ obsoletes_dist = 6
+ requires_external = 7
+ project_url = 8
+ by_val = {}
+for k,v in dependency.__dict__.items():
+ if not isinstance(v, int):
+ continue
+ dependency.by_val[v] = k
+
keep_conn = False
connection = None
keep_trove = True
@@ -257,7 +276,7 @@
# now see if we're inserting or updating a release
message = None
- relationships = {}
+ relationships = defaultdict(set)
old_cifiers = []
html = None
if self.has_release(name, version):
@@ -295,16 +314,17 @@
old.append('classifiers')
# get old classifiers list
- for col in ('requires', 'provides', 'obsoletes', 'requires_dist',
- 'provides_dist', 'obsoletes_dist',
- 'requires_external', 'project_url'):
- relationships[col] = self.get_release_relationships(name,
- version, col)
- relationships[col].sort()
- new_val = info.get(col, [])
- new_val.sort()
- if info.has_key(col) and relationships[col] != new_val:
- old.append(col)
+ for kind, specifier in self.get_release_dependencies(name, version):
+ relationships[kind].add(specifier)
+ for nkind, skind in dependency.by_val.items():
+ # numerical kinds in relationships; string kinds in info
+ try:
+ new_val = set(info[skind])
+ except KeyError:
+ # value not provided
+ continue
+ if relationships[skind] != new_val:
+ old.append(skind)
# no update when nothing changes
if not old:
@@ -385,17 +405,15 @@
(name, version, trove_id))
# handle relationship specifiers
- for col in ('requires', 'provides', 'obsoletes', 'requires_dist',
- 'provides_dist', 'obsoletes_dist',
- 'requires_external', 'project_url'):
- if not info.has_key(col) or relationships.get(col, []) == info[col]:
+ for nkind, skind in dependency.by_val.items():
+ if not info.has_key(skind) or relationships[nkind] == set(info[skind]):
continue
- safe_execute(cursor, '''delete from release_%s where name=%%s
- and version=%%s'''%col, (name, version))
- for specifier in info[col]:
- safe_execute(cursor, '''insert into release_%s (name, version,
- specifier) values (%%s, %%s, %%s)'''%col, (name,
- version, specifier))
+ safe_execute(cursor, '''delete from release_dependencies where name=%s
+ and version=%s and kind=%s''', (name, version, nkind))
+ for specifier in info[skind]:
+ safe_execute(cursor, '''insert into release_dependencies (name, version,
+ kind, specifier) values (%s, %s, %s, %s)''', (name,
+ version, nkind, specifier))
return message
@@ -667,10 +685,19 @@
"requires", "provides" or "obsoletes".
'''
cursor = self.get_cursor()
- safe_execute(cursor, '''select specifier from release_%s where
- name=%%s and version=%%s'''%relationship, (name, version))
+ safe_execute(cursor, '''select specifier from release_dependencies where
+ name=%s and version=%s and kind=%s''', (name, version,
+ getattr(dependency, relationship)))
return Result(None, cursor.fetchall(), self._Release_Relationships)
+ _Release_Dependencies = FastResultRow('kind! specifier')
+ def get_release_dependencies(self, name, version):
+ '''Fetch all release dependencies of a release.'''
+ cursor = self.get_cursor()
+ safe_execute(cursor, '''select kind, specifier from release_dependencies
+ where name=%s and version=%s''', (name, version))
+ return Result(None, cursor.fetchall(), self._Release_Dependencies)
+
_Package_Roles = FastResultRow('role_name user_name')
def get_package_roles(self, name):
''' Fetch the list of Roles for the package.
@@ -703,8 +730,9 @@
def get_package_requires_dist(self, name, version):
cursor = self.get_cursor()
- safe_execute(cursor, '''select specifier from release_requires_dist
- where name=%s and version=%s ''', (name, version))
+ safe_execute(cursor, '''select specifier from release_dependencies
+ where name=%s and version=%s and kind=%s''', (name, version,
+ dependency.requires_dist))
packages = []
for package in cursor.fetchall():
pack = {'name': package[0],
@@ -714,8 +742,9 @@
def get_package_provides_dist(self, name, version):
cursor = self.get_cursor()
- safe_execute(cursor, '''select specifier from release_provides_dist
- where name=%s and version=%s ''', (name, version))
+ safe_execute(cursor, '''select specifier from release_dependencies
+ where name=%s and version=%s and kind=%s''', (name, version,
+ dependency.provides_dist))
packages = []
for package in cursor.fetchall():
pack = {'name': package[0],
@@ -725,8 +754,9 @@
def get_package_obsoletes_dist(self, name, version):
cursor = self.get_cursor()
- safe_execute(cursor, '''select specifier from release_obsoletes_dist
- where name=%s and version=%s ''', (name, version))
+ safe_execute(cursor, '''select specifier from release_dependencies
+ where name=%s and version=%s and kind=%s''', (name, version,
+ dependency.obsoletes_dist))
packages = []
for package in cursor.fetchall():
pack = {'name': package[0],
@@ -736,14 +766,16 @@
def get_package_requires_external(self, name, version):
cursor = self.get_cursor()
- safe_execute(cursor, '''select specifier from release_requires_external
- where name=%s and version=%s ''', (name, version))
+ safe_execute(cursor, '''select specifier from release_dependencies
+ where name=%s and version=%s and kind=%s''', (name, version,
+ dependency.requires_external))
return [package[0] for package in cursor.fetchall()]
def get_package_project_url(self, name, version):
cursor = self.get_cursor()
- safe_execute(cursor, '''select specifier from release_project_url
- where name=%s and version=%s ''', (name, version))
+ safe_execute(cursor, '''select specifier from release_dependencies
+ where name=%s and version=%s and kind=%s''', (name, version,
+ dependency.project_url))
project_urls = []
for project in cursor.fetchall():
project_urls.append(project[0].split(','))
@@ -969,10 +1001,7 @@
file['filename']))
# delete ancillary table entries
- for tab in ('files', 'provides', 'requires', 'obsoletes',
- 'classifiers', 'requires_dist', 'provides_dist',
- 'obsoletes_dist', 'requires_external',
- 'project_url'):
+ for tab in ('files', 'dependencies', 'classifiers'):
safe_execute(cursor, '''delete from release_%s where
name=%%s and version=%%s'''%tab, (name, version))
safe_execute(cursor, 'delete from description_urls where name=%s and version=%s',
@@ -998,10 +1027,7 @@
file['filename']))
# delete ancillary table entries
- for tab in ('files', 'provides', 'requires', 'obsoletes',
- 'classifiers', 'requires_dist', 'provides_dist',
- 'obsoletes_dist', 'requires_external',
- 'project_url'):
+ for tab in ('files', 'dependencies', 'classifiers'):
safe_execute(cursor, 'delete from release_%s where name=%%s'%tab,
(name, ))
Modified: trunk/pypi/tools/demodata.py
==============================================================================
--- trunk/pypi/tools/demodata.py (original)
+++ trunk/pypi/tools/demodata.py Sat Aug 14 18:02:34 2010
@@ -48,6 +48,7 @@
'description':'Does anybody want to provide real data here?',
'classifiers':["Development Status :: 3 - Alpha",
"Programming Language :: Python :: 3"],
+ 'requires_dist':['spam'],
'_pypi_hidden':version!='0.4'
})
Added: trunk/pypi/tools/sql-migrate-20100814.sql
==============================================================================
--- (empty file)
+++ trunk/pypi/tools/sql-migrate-20100814.sql Sat Aug 14 18:02:34 2010
@@ -0,0 +1,41 @@
+BEGIN;
+CREATE TABLE release_dependencies (
+ name TEXT,
+ version TEXT,
+ kind INTEGER,
+ specifier TEXT,
+ FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE
+);
+grant all on release_dependencies to pypi;
+
+insert into release_dependencies(name, version, kind, specifier)
+ select name, version, 1, specifier from release_requires;
+insert into release_dependencies(name, version, kind, specifier)
+ select name, version, 2, specifier from release_provides;
+insert into release_dependencies(name, version, kind, specifier)
+ select name, version, 3, specifier from release_obsoletes;
+insert into release_dependencies(name, version, kind, specifier)
+ select name, version, 4, specifier from release_requires_dist;
+insert into release_dependencies(name, version, kind, specifier)
+ select name, version, 5, specifier from release_provides_dist;
+insert into release_dependencies(name, version, kind, specifier)
+ select name, version, 6, specifier from release_obsoletes_dist;
+insert into release_dependencies(name, version, kind, specifier)
+ select name, version, 7, specifier from release_requires_external;
+insert into release_dependencies(name, version, kind, specifier)
+ select name, version, 8, specifier from release_project_url;
+
+CREATE INDEX rel_dep_name_idx ON release_dependencies(name);
+CREATE INDEX rel_dep_name_version_idx ON release_dependencies(name, version);
+CREATE INDEX rel_dep_name_version_kind_idx ON release_dependencies(name, version, kind);
+
+drop table release_requires;
+drop table release_provides;
+drop table release_obsoletes;
+drop table release_requires_dist;
+drop table release_provides_dist;
+drop table release_obsoletes_dist;
+drop table release_requires_external;
+drop table release_project_url;
+
+COMMIT;
More information about the Pypi-checkins
mailing list