From python-checkins at python.org Wed Aug 17 18:03:20 2011 From: python-checkins at python.org (martin.von.loewis) Date: Wed, 17 Aug 2011 18:03:20 +0200 (CEST) Subject: [Pypi-checkins] r939 - trunk/pypi/templates Message-ID: <3RfFhm2xzxzMsc@mail.python.org> Author: martin.von.loewis Date: Wed Aug 17 18:03:20 2011 New Revision: 939 Modified: trunk/pypi/templates/display.pt Log: Remove note about rating removal. Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Wed Aug 17 18:03:20 2011 @@ -244,8 +244,6 @@ -The rating feature has been removed. See catalog-sig for the discussion of this removal. - From python-checkins at python.org Wed Aug 17 18:10:06 2011 From: python-checkins at python.org (martin.von.loewis) Date: Wed, 17 Aug 2011 18:10:06 +0200 (CEST) Subject: [Pypi-checkins] r940 - trunk/pypi Message-ID: <3RfFrZ2KfxzMtT@mail.python.org> Author: martin.von.loewis Date: Wed Aug 17 18:10:06 2011 New Revision: 940 Modified: trunk/pypi/standalone.py Log: Make standalone server redirect / to /pypi Modified: trunk/pypi/standalone.py ============================================================================== --- trunk/pypi/standalone.py (original) +++ trunk/pypi/standalone.py Wed Aug 17 18:10:06 2011 @@ -9,6 +9,10 @@ self.send_header('Content-Type', content_type) def run(self): + if self.path == '/': + self.send_response(301) + self.send_header('Location', '/pypi') + return for scriptname in ('/mirrors', '/simple', '/pypi', '/serversig', '/daytime'): if self.path.startswith(scriptname): rest = self.path[len(scriptname):] From python-checkins at python.org Wed Aug 17 19:16:39 2011 From: python-checkins at python.org (martin.von.loewis) Date: Wed, 17 Aug 2011 19:16:39 +0200 (CEST) Subject: [Pypi-checkins] r941 - in trunk/pypi: . tools Message-ID: <3RfHKM6ms2zN8K@mail.python.org> Author: martin.von.loewis Date: Wed Aug 17 19:16:39 2011 New Revision: 941 Added: trunk/pypi/tools/sql-migrate-20110831.sql (contents, props changed) Modified: trunk/pypi/store.py trunk/pypi/webui.py Log: Add user deletion facility. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Wed Aug 17 19:16:39 2011 @@ -1398,12 +1398,16 @@ return res[0] _User_Packages = FastResultRow('package_name') - def user_packages(self, user): + def user_packages(self, user, only_owner=False): ''' Retrieve package info for all packages of a user ''' cursor = self.get_cursor() + owner_sql = '' + if only_owner: + owner_sql = "and roles.role_name='Owner'" sql = '''select distinct(package_name),lower(package_name) from roles where roles.user_name=%s and package_name is not NULL + ''' + owner_sql + ''' order by lower(package_name)''' safe_execute(cursor, sql, (user,)) res = cursor.fetchall() @@ -1411,6 +1415,28 @@ res = [] return Result(None, res, self._User_Packages) + def delete_user(self, user): + '''Delete a user. Return None.''' + cursor = self.get_cursor() + # delete all maintainer roles + safe_execute(cursor, + '''delete from roles where role_name='Maintainer' + and user_name=%s''', + (user,)) + # point all journal entries to the "deleted user" + safe_execute(cursor, + '''update journals set name='deleted user' where name=%s''', + (user,)) + # delete all cookies + safe_execute(cursor, + '''delete from cookies where name=%s''', + (user,)) + # every other reference should either be cascading, + # or it's a bug to break it + + # delete user account itself + safe_execute(cursor, 'delete from users where name=%s', (user,)) + # # Trove # Added: trunk/pypi/tools/sql-migrate-20110831.sql ============================================================================== --- (empty file) +++ trunk/pypi/tools/sql-migrate-20110831.sql Wed Aug 17 19:16:39 2011 @@ -0,0 +1,2 @@ +insert into users(name, password, email, gpg_keyid, last_login) +values('deleted user', 'invalid', '', '', '2000-01-01'); Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Wed Aug 17 19:16:39 2011 @@ -520,7 +520,7 @@ display register_form user_form forgotten_password_form user password_reset role role_form list_classifiers login logout files file_upload show_md5 doc_upload claim openid openid_return dropid - clear_auth addkey delkey lasthour json gae_file about'''.split(): + clear_auth addkey delkey lasthour json gae_file about delete_user'''.split(): getattr(self, action)() else: #raise NotFound, 'Unknown action %s' % action @@ -2314,6 +2314,7 @@ if self.username: user = self.store.get_user(self.username) info['new_user'] = False + info['owns_packages'] = bool(self.store.user_packages(self.username, True)) info['name'] = user['name'] info['email'] = user['email'] info['action'] = 'Update details' @@ -2571,6 +2572,31 @@ self.write_template("password_reset.pt", title="Request password reset", retry=True) + def delete_user(self): + if not self.authenticated: + raise Unauthorised + if self.form.has_key('submit_ok'): + # ok, do it + self.store.delete_user(self.username) + self.authenticated = self.loggedin = False + self.username = self.usercookie = None + return self.home() + elif self.form.has_key('submit_cancel'): + self.ok_message='Deletion cancelled' + return self.home() + else: + message = '''You are about to delete the %s account
+ This action cannot be undone!
+ Are you sure?'''%self.username + + fields = [ + {'name': ':action', 'value': 'delete_user'}, + ] + return self.write_template('dialog.pt', message=message, + title='Confirm account deletion', fields=fields) + + + def send_email(self, recipient, message): ''' Send an administrative email to the recipient ''' From python-checkins at python.org Wed Aug 17 19:20:37 2011 From: python-checkins at python.org (martin.von.loewis) Date: Wed, 17 Aug 2011 19:20:37 +0200 (CEST) Subject: [Pypi-checkins] r942 - trunk/pypi/templates Message-ID: <3RfHPx15zVzM5D@mail.python.org> Author: martin.von.loewis Date: Wed Aug 17 19:20:36 2011 New Revision: 942 Modified: trunk/pypi/templates/register.pt Log: Support user deletion. Modified: trunk/pypi/templates/register.pt ============================================================================== --- trunk/pypi/templates/register.pt (original) +++ trunk/pypi/templates/register.pt Wed Aug 17 19:20:36 2011 @@ -5,12 +5,22 @@ metal:use-macro="standard_template/macros/page"> -

+

This form allows "traditional" registration (using a password). Users who want to register with their OpenID (e.g. Google, myOpenID or Launchpad account) should follow one of the links to the right.

+

+ This form allows to modify your PyPI account information, and to delete your account. +
+ + + + You cannot delete your account since you + are still listed as owner of some packages. +

Author: richard Date: Mon Aug 22 03:19:57 2011 New Revision: 943 Added: trunk/pypi/tools/sql-migrate-20110822.sql Modified: trunk/pypi/pkgbase_schema.sql Log: fix FK contstraint handling on user delete; thanks Andy Todd Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Mon Aug 22 03:19:57 2011 @@ -13,7 +13,7 @@ CREATE TABLE openids ( id TEXT PRIMARY KEY, - name TEXT REFERENCES users + name TEXT REFERENCES users ON DELETE CASCADE ); CREATE TABLE openid_sessions ( Added: trunk/pypi/tools/sql-migrate-20110822.sql ============================================================================== --- (empty file) +++ trunk/pypi/tools/sql-migrate-20110822.sql Mon Aug 22 03:19:57 2011 @@ -0,0 +1,2 @@ +ALTER TABLE openids DROP CONSTRAINT openids_name_fkey; +ALTER TABLE openids ADD CONSTRAINT openids_name_fkey FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; From python-checkins at python.org Mon Aug 22 03:23:06 2011 From: python-checkins at python.org (richard) Date: Mon, 22 Aug 2011 03:23:06 +0200 (CEST) Subject: [Pypi-checkins] r944 - trunk/pypi Message-ID: <3Rhxwp5tyfzNJY@mail.python.org> Author: richard Date: Mon Aug 22 03:23:06 2011 New Revision: 944 Modified: trunk/pypi/webui.py Log: handle more form input str/unicode fun; thanks Capel Brunker Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Mon Aug 22 03:23:06 2011 @@ -1063,6 +1063,8 @@ write_element(pelem, person, 'foaf:name') email = info[person+'_email'] if email and email != 'UNKNOWN': + # sha1 requires ascii not unicode + if isinstance(email, unicode): email = email.encode('utf8') obj = hashlib.sha1(email) email = binascii.b2a_hex(obj.digest()) elem = SE(pelem, 'foaf:mbox_sha1sum') From python-checkins at python.org Mon Aug 22 03:29:37 2011 From: python-checkins at python.org (richard) Date: Mon, 22 Aug 2011 03:29:37 +0200 (CEST) Subject: [Pypi-checkins] r945 - trunk/pypi Message-ID: <3Rhy4K3NWTzN6r@mail.python.org> Author: richard Date: Mon Aug 22 03:29:37 2011 New Revision: 945 Modified: trunk/pypi/webui.py Log: handle bare values uploaded as file content; thanks Capel Brunker Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Mon Aug 22 03:29:37 2011 @@ -2234,7 +2234,13 @@ if not self.form.has_key('content'): raise FormError, "No file uploaded" - data = self.form['content'].value + try: + data = self.form['content'].value + except AttributeError: + # error trying to get the .value *probably* means we didn't get + # a file uploaded in the content element + raise FormError, "No file uploaded" + if len(data) > 10*1024*1024: raise FormError, "Documentation zip file is too large" data = cStringIO.StringIO(data) From python-checkins at python.org Mon Aug 22 07:33:15 2011 From: python-checkins at python.org (richard) Date: Mon, 22 Aug 2011 07:33:15 +0200 (CEST) Subject: [Pypi-checkins] r946 - trunk/pypi Message-ID: <3Rj3TR2P6BzNBN@mail.python.org> Author: richard Date: Mon Aug 22 07:33:15 2011 New Revision: 946 Modified: trunk/pypi/pkgbase_schema.sql Log: update with current production schema Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Mon Aug 22 07:33:15 2011 @@ -244,7 +244,13 @@ -- Table structure for table: mirrors CREATE TABLE mirrors ( ip TEXT PRIMARY KEY, - user_name TEXT REFERENCES users + user_name TEXT REFERENCES users, + index_url TEXT, + last_modified_url TEXT, + local_stats_url TEXT, + stats_url TEXT, + mirrors_url TEXT + ); -- ratings From python-checkins at python.org Mon Aug 22 07:38:18 2011 From: python-checkins at python.org (richard) Date: Mon, 22 Aug 2011 07:38:18 +0200 (CEST) Subject: [Pypi-checkins] r947 - trunk/pypi Message-ID: <3Rj3bG1qlGzNKN@mail.python.org> Author: richard Date: Mon Aug 22 07:38:18 2011 New Revision: 947 Modified: trunk/pypi/pkgbase_schema.sql Log: hmm, the most recent migrate with the referenced columns removes them, even though they are in the production schema... Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Mon Aug 22 07:38:18 2011 @@ -244,13 +244,7 @@ -- Table structure for table: mirrors CREATE TABLE mirrors ( ip TEXT PRIMARY KEY, - user_name TEXT REFERENCES users, - index_url TEXT, - last_modified_url TEXT, - local_stats_url TEXT, - stats_url TEXT, - mirrors_url TEXT - + user_name TEXT REFERENCES users ); -- ratings From python-checkins at python.org Mon Aug 22 07:42:46 2011 From: python-checkins at python.org (richard) Date: Mon, 22 Aug 2011 07:42:46 +0200 (CEST) Subject: [Pypi-checkins] r948 - trunk/pypi Message-ID: <3Rj3hQ1Wn6zMtV@mail.python.org> Author: richard Date: Mon Aug 22 07:42:46 2011 New Revision: 948 Modified: trunk/pypi/rpc.py trunk/pypi/store.py Log: add XML-RPC interfaces for retrieving download counts for releases, packages by user and users (roles) by package Modified: trunk/pypi/rpc.py ============================================================================== --- trunk/pypi/rpc.py (original) +++ trunk/pypi/rpc.py Mon Aug 22 07:42:46 2011 @@ -25,6 +25,9 @@ self.register_function(changelog) self.register_function(changed_packages) self.register_function(post_cheesecake_for_release) + self.register_function(release_downloads) + self.register_function(package_roles) + self.register_function(user_packages) self.register_introspection_functions() self.register_multicall_functions() @@ -52,6 +55,20 @@ raise Fault, "multicall too large" return SimpleXMLRPCDispatcher.system_multicall(self, call_list) +def release_downloads(store, package_name, version): + '''Return download count for given release.''' + return store.get_release_downloads(package_name, version) + +def package_roles(store, package_name): + '''Return associated users and package roles.''' + result = store.get_package_roles(package_name) + return [tuple(fields.values())for fields in result] + +def user_packages(store, user): + '''Return associated packages for user.''' + result = store.get_user_packages(user) + return [tuple(fields.values()) for fields in result] + def list_packages(store): result = store.get_packages() return [row['name'] for row in result] Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Mon Aug 22 07:42:46 2011 @@ -705,6 +705,21 @@ where name=%s and version=%s''', (name, version)) return Result(None, cursor.fetchall(), self._Release_Dependencies) + def get_release_downloads(self, name, version): + '''Fetch current download count for a release.''' + cursor = self.get_cursor() + safe_execute(cursor, '''select filename, downloads from release_files where + name=%s and version=%s''', (name, version)) + return cursor.fetchall() + + _Package_Roles = FastResultRow('role_name package_name') + def get_user_packages(self, name): + '''Fetch all packages and roles associated to user.''' + cursor = self.get_cursor() + safe_execute(cursor, '''select role_name, package_name from roles where + user_name=%s''', (name,)) + return Result(None, cursor.fetchall(), self._Package_Roles) + _Package_Roles = FastResultRow('role_name user_name') def get_package_roles(self, name): ''' Fetch the list of Roles for the package. From python-checkins at python.org Mon Aug 22 08:51:59 2011 From: python-checkins at python.org (richard) Date: Mon, 22 Aug 2011 08:51:59 +0200 (CEST) Subject: [Pypi-checkins] r949 - in trunk/pypi: . templates Message-ID: <3Rj5DH213SzNDq@mail.python.org> Author: richard Date: Mon Aug 22 08:51:58 2011 New Revision: 949 Added: trunk/pypi/templates/packages-rss.xml Modified: trunk/pypi/config.py trunk/pypi/store.py trunk/pypi/webui.py Log: add newest packages RSS Modified: trunk/pypi/config.py ============================================================================== --- trunk/pypi/config.py (original) +++ trunk/pypi/config.py Mon Aug 22 08:51:58 2011 @@ -30,6 +30,7 @@ self.simple_script = c.get('webui', 'simple_script') self.files_url = c.get('webui', 'files_url') self.rss_file = c.get('webui', 'rss_file') + self.packages_rss_file = c.get('webui', 'packages_rss_file') self.debug_mode = c.get('webui', 'debug_mode') self.cheesecake_password = c.get('webui', 'cheesecake_password') self.key_dir = c.get('webui', 'key_dir') Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Mon Aug 22 08:51:58 2011 @@ -884,12 +884,13 @@ # Postgres will only do that if the number of expected results # is "small". statement = ''' - select j.name, j.version, j.submitted_date, r.summary - from (select name,version,submitted_date from journals - where version is not null and action='create' - order by submitted_date desc %s) j, releases r - where j.name=r.name and j.version=r.version - and not r._pypi_hidden order by j.submitted_date desc''' + select j.name, r.version, j.submitted_date, r.summary + from (select name,version,submitted_date from journals + where action='create' order by submitted_date desc %s) j, + releases r + where j.name=r.name and r.version is not NULL + and not r._pypi_hidden + order by j.submitted_date desc''' #print ' '.join((statement % limit).split()) safe_execute(cursor, statement % limit) result = Result(None, self.get_unique(cursor.fetchall())[:num], Added: trunk/pypi/templates/packages-rss.xml ============================================================================== --- (empty file) +++ trunk/pypi/templates/packages-rss.xml Mon Aug 22 08:51:58 2011 @@ -0,0 +1,20 @@ + + + + + PyPI Newest Packages + + Newest packages registered at the Python Package Index + en + + + + <link tal:content="python:'http://pypi.python.org%s'%app.packageURL( + release['name'], release['version'])" /> + <description tal:content="release/summary" /> + <pubDate tal:content="python:release['submitted_date'].strftime('%d %b %Y %H:%M:%S GMT')" /> + </item> + </channel> +</rss> Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Mon Aug 22 08:51:58 2011 @@ -389,7 +389,8 @@ ('submit_form', 'Package submission'), ('list_classifiers', 'List trove classifiers'), ('index', 'List packages'), - ('rss', 'RSS (last 40 updates)'), + ('rss', 'RSS (latest 40 updates)'), + ('package_rss', 'RSS (newest 40 packages)'), ('role_form', 'Admin'), ) def navlinks_html(self): @@ -520,7 +521,8 @@ display register_form user_form forgotten_password_form user password_reset role role_form list_classifiers login logout files file_upload show_md5 doc_upload claim openid openid_return dropid - clear_auth addkey delkey lasthour json gae_file about delete_user'''.split(): + clear_auth addkey delkey lasthour json gae_file about delete_user + rss_regen'''.split(): getattr(self, action)() else: #raise NotFound, 'Unknown action %s' % action @@ -677,7 +679,7 @@ """ # determine whether the rss file is up to date if not os.path.exists(self.config.rss_file): - self.rss_regen(self.config.rss_file) + self.rss_regen() # TODO: throw in a last-modified header too? self.handler.send_response(200, 'OK') @@ -685,22 +687,49 @@ self.handler.end_headers() self.wfile.write(open(self.config.rss_file).read()) - def rss_regen(self, rss_file=None): - if rss_file is None: - rss_file = self.config.rss_file + def packages_rss(self): + """Dump the last N days' updates as an RSS feed. + """ + # determine whether the rss file is up to date + if not os.path.exists(self.config.packages_rss_file): + self.rss_regen() + + # TODO: throw in a last-modified header too? + self.handler.send_response(200, 'OK') + self.handler.set_content_type('text/xml; charset=utf-8') + self.handler.end_headers() + self.wfile.write(open(self.config.packages_rss_file).read()) + + def rss_regen(self): context = {} context['app'] = self + # generate the releases RSS template_dir = os.path.join(os.path.dirname(__file__), 'templates') template = PyPiPageTemplate('rss.xml', template_dir) content = template(**context) + f = open(self.config.rss_file, 'w') + try: + f.write(content.encode('utf-8')) + finally: + f.close() - f = open(rss_file, 'w') + # generate the packages RSS + template_dir = os.path.join(os.path.dirname(__file__), 'templates') + template = PyPiPageTemplate('packages-rss.xml', template_dir) + content = template(**context) + f = open(self.config.packages_rss_file, 'w') try: f.write(content.encode('utf-8')) finally: f.close() + # just for making this all OK + self.handler.send_response(200, 'OK') + self.handler.set_content_type('text/plain') + self.handler.end_headers() + self.wfile.write('OK') + def lasthour(self): self.write_template('rss1hour.xml', **{'content-type':'text/xml; charset=utf-8'}) From python-checkins at python.org Mon Aug 22 08:55:52 2011 From: python-checkins at python.org (richard) Date: Mon, 22 Aug 2011 08:55:52 +0200 (CEST) Subject: [Pypi-checkins] r950 - trunk/pypi Message-ID: <3Rj5Jm0pZrzNDq@mail.python.org> Author: richard Date: Mon Aug 22 08:55:51 2011 New Revision: 950 Modified: trunk/pypi/webui.py Log: fix Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Mon Aug 22 08:55:51 2011 @@ -390,7 +390,7 @@ ('list_classifiers', 'List trove classifiers'), ('index', 'List packages'), ('rss', 'RSS (latest 40 updates)'), - ('package_rss', 'RSS (newest 40 packages)'), + ('packages_rss', 'RSS (newest 40 packages)'), ('role_form', 'Admin'), ) def navlinks_html(self): From python-checkins at python.org Mon Aug 22 12:02:32 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 22 Aug 2011 12:02:32 +0200 (CEST) Subject: [Pypi-checkins] r951 - trunk/pypi Message-ID: <3Rj9S83QZJzNLn@mail.python.org> Author: martin.von.loewis Date: Mon Aug 22 12:02:32 2011 New Revision: 951 Modified: trunk/pypi/store.py Log: Fix journal update. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Mon Aug 22 12:02:32 2011 @@ -1441,12 +1441,12 @@ (user,)) # point all journal entries to the "deleted user" safe_execute(cursor, - '''update journals set name='deleted user' where name=%s''', + '''update journals set submitted_by='deleted user' where submitted_by=%s''', (user,)) # delete all cookies safe_execute(cursor, '''delete from cookies where name=%s''', - (user,)) + (user,)) # every other reference should either be cascading, # or it's a bug to break it From python-checkins at python.org Mon Aug 22 14:40:52 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 22 Aug 2011 14:40:52 +0200 (CEST) Subject: [Pypi-checkins] r952 - trunk/pypi/templates Message-ID: <3RjDyr5nb0zNMN@mail.python.org> Author: martin.von.loewis Date: Mon Aug 22 14:40:52 2011 New Revision: 952 Modified: trunk/pypi/templates/display.pt Log: Issue 3388335: Don't render metadata fields as HTML except for the ones known to be safe. Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Mon Aug 22 14:40:52 2011 @@ -133,29 +133,28 @@ <li tal:condition="data/release/license | nothing"> <strong>License:</strong> <tal:block define="hasnl python:'\n' in data['release']['license']"> - <pre tal:condition="hasnl" tal:content="structure data/release/license" /> + <pre tal:condition="hasnl" tal:content="data/release/license" /> <span tal:condition="not:hasnl" tal:content="data/release/license" /> </tal:block> </li> <li tal:condition="data/release/platform | nothing"> <strong>Platform:</strong> - <span tal:content="structure - python:data['release']['platform'].replace('\n', ', ')" /> + <span tal:content="python:data['release']['platform'].replace('\n', ', ')" /> </li> <!-- TODO: add link to products in follow dependencies... --> <li tal:condition="data/requires | nothing"> <strong>Requires</strong> - <tal:element replace="structure python:', '.join(data['requires'])" /> + <tal:element replace="python:', '.join(data['requires'])" /> </li> <li tal:condition="data/provides | nothing"> <strong>Provides</strong> - <tal:element replace="structure python:', '.join(data['provides'])" /> + <tal:element replace="python:', '.join(data['provides'])" /> </li> <li tal:condition="data/obsoletes | nothing"> <strong>Obsoletes</strong> - <tal:element replace="structure python:', '.join(data['obsoletes'])" /> + <tal:element replace="python:', '.join(data['obsoletes'])" /> </li> <li tal:condition="data/categories | nothing"> From python-checkins at python.org Tue Aug 23 02:44:55 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 02:44:55 +0200 (CEST) Subject: [Pypi-checkins] r953 - trunk/pypi/templates Message-ID: <3RjY2H01z3zNKp@mail.python.org> Author: richard Date: Tue Aug 23 02:44:54 2011 New Revision: 953 Modified: trunk/pypi/templates/standard_template.pt Log: add daily rss to header Modified: trunk/pypi/templates/standard_template.pt ============================================================================== --- trunk/pypi/templates/standard_template.pt (original) +++ trunk/pypi/templates/standard_template.pt Tue Aug 23 02:44:54 2011 @@ -11,7 +11,8 @@ <title tal:content="string:${data/title} : Python Package Index" /> <meta tal:attributes="content data/keywords" /> <meta tal:attributes="content data/description" /> - <link rel="alternate" type="application/rss+xml" title="RSS: 30 latest updates" href="http://www.python.org/pypi?:action=rss"/> + <link rel="alternate" type="application/rss+xml" title="RSS: 40 latest updates" href="http://www.python.org/pypi?:action=rss"/> + <link rel="alternate" type="application/rss+xml" title="RSS: 40 newest packages" href="http://www.python.org/pypi?:action=packages_rss"/> <link media="screen" tal:attributes="href string:${app/config/pydotorg}styles/screen-switcher-default.css" type="text/css" id="screen-switcher-stylesheet" rel="stylesheet" /> <link media="screen" tal:attributes="href string:${app/config/pydotorg}styles/netscape4.css" type="text/css" rel="stylesheet" /> <link media="print" tal:attributes="href string:${app/config/pydotorg}styles/print.css" type="text/css" rel="stylesheet" /> From python-checkins at python.org Tue Aug 23 03:13:55 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 03:13:55 +0200 (CEST) Subject: [Pypi-checkins] r954 - trunk/pypi Message-ID: <3RjYgl2NJczNNd@mail.python.org> Author: richard Date: Tue Aug 23 03:13:55 2011 New Revision: 954 Modified: trunk/pypi/store.py Log: better newest package SQL; thanks Andy Todd Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Tue Aug 23 03:13:55 2011 @@ -885,11 +885,14 @@ # is "small". statement = ''' select j.name, r.version, j.submitted_date, r.summary - from (select name,version,submitted_date from journals - where action='create' order by submitted_date desc %s) j, - releases r - where j.name=r.name and r.version is not NULL + from releases r + JOIN (SELECT name, max(submitted_Date) submitted_date + FROM journals GROUP BY name) j ON j.name = r.name + where r.version is not NULL and not r._pypi_hidden + and r.name in (SELECT name FROM journals + WHERE action='create' + ORDER BY submitted_date DESC %s) order by j.submitted_date desc''' #print ' '.join((statement % limit).split()) safe_execute(cursor, statement % limit) From python-checkins at python.org Tue Aug 23 03:23:14 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 03:23:14 +0200 (CEST) Subject: [Pypi-checkins] r955 - trunk/pypi Message-ID: <3RjYtV08BgzNPK@mail.python.org> Author: richard Date: Tue Aug 23 03:23:13 2011 New Revision: 955 Modified: trunk/pypi/webui.py Log: whoops Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Aug 23 03:23:13 2011 @@ -724,12 +724,6 @@ finally: f.close() - # just for making this all OK - self.handler.send_response(200, 'OK') - self.handler.set_content_type('text/plain') - self.handler.end_headers() - self.wfile.write('OK') - def lasthour(self): self.write_template('rss1hour.xml', **{'content-type':'text/xml; charset=utf-8'}) From python-checkins at python.org Tue Aug 23 08:03:30 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 08:03:30 +0200 (CEST) Subject: [Pypi-checkins] r956 - in trunk/pypi: . templates tools Message-ID: <3Rjh5t6VM2zNRG@mail.python.org> Author: richard Date: Tue Aug 23 08:03:30 2011 New Revision: 956 Added: trunk/pypi/tools/sql-migrate-20110823.sql Modified: trunk/pypi/pkgbase_schema.sql trunk/pypi/store.py trunk/pypi/templates/display.pt trunk/pypi/webui.py Log: add ability for packages to list a bug tracker stored against the package (not release); thanks Kaleb Ufton Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Tue Aug 23 08:03:30 2011 @@ -85,6 +85,7 @@ name TEXT PRIMARY KEY, stable_version TEXT, normalized_name TEXT, + bugtrack_url TEXT, autohide BOOLEAN DEFAULT TRUE, comments BOOLEAN DEFAULT TRUE ); Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Tue Aug 23 08:03:30 2011 @@ -250,8 +250,18 @@ # see if we're inserting or updating a package if not self.has_package(name): # insert the new package entry - sql = 'insert into packages (name, normalized_name) values (%s, %s)' - safe_execute(cursor, sql, (name, normalize_package_name(name))) + cols = 'name, normalized_name' + vals = '%s, %s' + args = (name, normalize_package_name(name)) + + # if a bugtracker url is provided then insert it too + if 'bugtrack_url' in info: + cols += ', bugtrack_url' + vals += ', %s' + args += (info['bugtrack_url'], ) + + sql = 'insert into packages (%s) values (%s)' % (cols, vals) + safe_execute(cursor, sql, args) # journal entry safe_execute(cursor, '''insert into journals (name, version, action, @@ -285,7 +295,7 @@ # handle the special vars that most likely won't have been # submitted - for k in ('_pypi_ordering', '_pypi_hidden'): + for k in ('_pypi_ordering', '_pypi_hidden', 'bugtrack_url'): if not info.has_key(k): info[k] = existing[k] @@ -307,6 +317,14 @@ vals.append(info[k]) vals.extend([name, version]) + # pull out the bugtrack_url and put it in the packages table + # instead + if 'bugtrack_url' in cols: + sql = 'update packages set bugtrack_url=%s where name=%s' + safe_execute(cursor, sql, (info['bugtrack_url'], name)) + del vals[cols.index('bugtrack_url')] + cols.remove('bugtrack_url') + # get old classifiers list old_cifiers = self.get_release_classifiers(name, version) old_cifiers.sort() @@ -507,9 +525,9 @@ _Package = FastResultRow('''name stable_version version author author_email maintainer maintainer_email home_page license summary description - description_html keywords platform requires_python download_url - _pypi_ordering! _pypi_hidden! cheesecake_installability_id! - cheesecake_documentation_id! cheesecake_code_kwalitee_id!''') + description_html keywords platform requires_python download_url + _pypi_ordering! _pypi_hidden! cheesecake_installability_id! + cheesecake_documentation_id! cheesecake_code_kwalitee_id! bugtrack_url!''') def get_package(self, name, version): ''' Retrieve info about the package from the database. @@ -520,10 +538,10 @@ author_email, maintainer, maintainer_email, home_page, license, summary, description, description_html, keywords, platform, requires_python, download_url, _pypi_ordering, - _pypi_hidden, + _pypi_hidden, cheesecake_installability_id, cheesecake_documentation_id, - cheesecake_code_kwalitee_id + cheesecake_code_kwalitee_id, bugtrack_url from packages, releases where packages.name=%s and version=%s and packages.name = releases.name''' Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Tue Aug 23 08:03:30 2011 @@ -117,6 +117,14 @@ tal:content="data/release/home_page" /> </li> + + <li tal:condition="data/bugtrack_url | nothing"> + <strong>Bug Tracker:</strong> + <a tal:attributes="href python:data['bugtrack_url']" + tal:content="structure + python:data['bugtrack_url'].replace('\n', ', ')"/> + </li> + <li tal:condition="data/release/download_url | nothing"> <strong>Download URL:</strong> <!-- <th>Download URL --> Added: trunk/pypi/tools/sql-migrate-20110823.sql ============================================================================== --- (empty file) +++ trunk/pypi/tools/sql-migrate-20110823.sql Tue Aug 23 08:03:30 2011 @@ -0,0 +1 @@ +alter table packages add bugtrack_url text; Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Aug 23 08:03:30 2011 @@ -1301,9 +1301,10 @@ columns = ('name version author author_email maintainer ' 'maintainer_email home_page requires_python download_url ' 'summary license description description_html keywords ' - 'platform').split() + 'platform bugtrack_url').split() release = {'description_html': ''} + bugtrack_url ='' for column in columns: value = info[column] if not info[column]: continue @@ -1326,6 +1327,9 @@ elif column.startswith('cheesecake_'): column = column[:-3] value = self.store.get_cheesecake_index(int(value)) + elif column == 'bugtrack_url': + bugtrack_url = value + value = info[column] release[column] = value roles = {} @@ -1392,6 +1396,7 @@ obsoletes_dist=obsoletes_dist, requires_external=requires_external, project_url=project_url, + bugtrack_url = bugtrack_url, requires_python=release.get('requires_python', '')) def index(self, nav_current='index', releases=None): @@ -1536,7 +1541,7 @@ w = content.write # display all the properties - for property in 'name version author author_email maintainer maintainer_email home_page license summary description keywords platform download_url _pypi_hidden'.split(): + for property in 'name version author author_email maintainer maintainer_email home_page license summary description keywords platform download_url _pypi_hidden bugtrack_url'.split(): # get the existing entry if self.form.has_key(property): value = self.form[property] From python-checkins at python.org Tue Aug 23 08:36:12 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 08:36:12 +0200 (CEST) Subject: [Pypi-checkins] r957 - trunk/pypi/tools Message-ID: <3Rjhqc6GCszNQx@mail.python.org> Author: richard Date: Tue Aug 23 08:36:12 2011 New Revision: 957 Added: trunk/pypi/tools/create_all_fks.sql trunk/pypi/tools/drop_all_fks.sql Log: sql from Andy Todd for managing FKs when attempting to import a package database dump; assumes older redundant FKs have already been dropped Added: trunk/pypi/tools/create_all_fks.sql ============================================================================== --- (empty file) +++ trunk/pypi/tools/create_all_fks.sql Tue Aug 23 08:36:12 2011 @@ -0,0 +1,55 @@ +begin; +-- OpenID tables + +ALTER TABLE openids ADD CONSTRAINT openids_name_fkey FOREIGN KEY (name) REFERENCES users (name) ON DELETE CASCADE; + +ALTER TABLE openid_stypes ADD CONSTRAINT openid_stypes_id_fkey FOREIGN KEY (id) REFERENCES openid_sessions ON DELETE CASCADE; + +ALTER TABLE cookies ADD CONSTRAINT cookies_name_fkey FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; + +ALTER TABLE sshkeys ADD CONSTRAINT sshkeys_name_fkey FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; + +ALTER TABLE rego_otk ADD CONSTRAINT rego_otk_name_fkey FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; + +ALTER TABLE journals ADD CONSTRAINT journals_submitted_by_fkey FOREIGN KEY (submitted_by) REFERENCES users ON DELETE CASCADE; + +ALTER TABLE cheesecake_subindices ADD CONSTRAINT cheesecake_subindices_main_index_id_fkey FOREIGN KEY (main_index_id) REFERENCES cheesecake_main_indices; + +ALTER TABLE releases ADD CONSTRAINT releases_name_fkey FOREIGN KEY (name) REFERENCES packages ON DELETE CASCADE; + +ALTER TABLE releases ADD CONSTRAINT releases_cheesecake_installability_id_fkey FOREIGN KEY (cheesecake_installability_id) REFERENCES cheesecake_main_indices; + +ALTER TABLE releases ADD CONSTRAINT releases_cheesecake_documentation_id_fkey FOREIGN KEY (cheesecake_documentation_id) REFERENCES cheesecake_main_indices; + +ALTER TABLE releases ADD CONSTRAINT releases_cheesecake_code_kwalitee_id_fkey FOREIGN KEY (cheesecake_code_kwalitee_id) REFERENCES cheesecake_main_indices; + +ALTER TABLE release_classifiers ADD CONSTRAINT release_classifiers_trove_id_fkey FOREIGN KEY (trove_id) REFERENCES trove_classifiers; + +ALTER TABLE release_classifiers ADD CONSTRAINT release_classifiers_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version); + +ALTER TABLE release_dependencies ADD CONSTRAINT release_dependencies_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE; + +ALTER TABLE release_files ADD CONSTRAINT release_files_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE; + +ALTER TABLE release_urls ADD CONSTRAINT release_urls_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE; + +ALTER TABLE description_urls ADD CONSTRAINT description_urls_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE; + +ALTER TABLE roles ADD CONSTRAINT roles_user_name_fkey FOREIGN KEY (user_name) REFERENCES users; +ALTER TABLE roles ADD CONSTRAINT roles_package_name_fkey FOREIGN KEY (package_name) REFERENCES packages ON UPDATE CASCADE; + +ALTER TABLE mirrors ADD CONSTRAINT mirrors_user_name_fkey FOREIGN KEY (user_name) REFERENCES users; + +ALTER TABLE ratings ADD CONSTRAINT ratings_user_name_fkey FOREIGN KEY (user_name) REFERENCES users ON DELETE CASCADE; + +ALTER TABLE ratings ADD CONSTRAINT ratings_name_fkey FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE; + +ALTER TABLE comments ADD CONSTRAINT comments_rating_fkey FOREIGN KEY (rating) REFERENCES ratings (id) ON DELETE CASCADE; + +ALTER TABLE comments ADD CONSTRAINT comments_user_name_fkey FOREIGN KEY (user_name) REFERENCES users ON DELETE CASCADE; + +ALTER TABLE comments ADD CONSTRAINT comments_in_reply_to_fkey FOREIGN KEY (in_reply_to) REFERENCES comments ON DELETE CASCADE; + +ALTER TABLE comments_journal ADD CONSTRAINT comments_journal_submitted_by_fkey FOREIGN KEY (submitted_by) REFERENCES users ON DELETE CASCADE; + +ALTER TABLE comments_journal ADD CONSTRAINT comments_journal_name_fkey FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE; Added: trunk/pypi/tools/drop_all_fks.sql ============================================================================== --- (empty file) +++ trunk/pypi/tools/drop_all_fks.sql Tue Aug 23 08:36:12 2011 @@ -0,0 +1,35 @@ +/* + +File Name : drop_all_fks.sql +Description: Drop all foreign keys for a PYPI database to allow pg_restore to + be run and clone the database +*/ + +ALTER TABLE openids DROP CONSTRAINT openids_name_fkey; +ALTER TABLE openid_stypes DROP CONSTRAINT openid_stypes_id_fkey; +ALTER TABLE cookies DROP CONSTRAINT cookies_name_fkey; +ALTER TABLE sshkeys DROP CONSTRAINT sshkeys_name_fkey; +ALTER TABLE rego_otk DROP CONSTRAINT rego_otk_name_fkey; +ALTER TABLE journals DROP CONSTRAINT journals_submitted_by_fkey; +ALTER TABLE cheesecake_subindices DROP CONSTRAINT cheesecake_subindices_main_index_id_fkey; +ALTER TABLE releases DROP CONSTRAINT releases_name_fkey; +ALTER TABLE releases DROP CONSTRAINT releases_cheesecake_installability_id_fkey; +ALTER TABLE releases DROP CONSTRAINT releases_cheesecake_documentation_id_fkey; +ALTER TABLE releases DROP CONSTRAINT releases_cheesecake_code_kwalitee_id_fkey; +ALTER TABLE release_classifiers DROP CONSTRAINT release_classifiers_trove_id_fkey; +ALTER TABLE release_classifiers DROP CONSTRAINT release_classifiers_name_fkey; +ALTER TABLE release_dependencies DROP CONSTRAINT release_dependencies_name_fkey; +ALTER TABLE release_files DROP CONSTRAINT release_files_name_fkey; +ALTER TABLE release_urls DROP CONSTRAINT release_urls_name_fkey; +ALTER TABLE description_urls DROP CONSTRAINT description_urls_name_fkey; +ALTER TABLE roles DROP CONSTRAINT roles_user_name_fkey; +ALTER TABLE roles DROP CONSTRAINT roles_package_name_fkey; +ALTER TABLE mirrors DROP CONSTRAINT mirrors_user_name_fkey; +ALTER TABLE ratings DROP CONSTRAINT ratings_user_name_fkey; +ALTER TABLE ratings DROP CONSTRAINT ratings_name_fkey; +ALTER TABLE comments DROP CONSTRAINT comments_rating_fkey; +ALTER TABLE comments DROP CONSTRAINT comments_user_name_fkey; +ALTER TABLE comments DROP CONSTRAINT comments_in_reply_to_fkey; +ALTER TABLE comments_journal DROP CONSTRAINT comments_journal_submitted_by_fkey; +ALTER TABLE comments_journal DROP CONSTRAINT comments_journal_name_fkey; + From python-checkins at python.org Tue Aug 23 08:44:44 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 08:44:44 +0200 (CEST) Subject: [Pypi-checkins] r958 - trunk/pypi Message-ID: <3Rjj1S5sGYzNGf@mail.python.org> Author: richard Date: Tue Aug 23 08:44:44 2011 New Revision: 958 Modified: trunk/pypi/pkgbase_schema.sql trunk/pypi/store.py trunk/pypi/webui.py Log: addition of CSRF protection; thanks Capel Brunker Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Tue Aug 23 08:44:44 2011 @@ -278,4 +278,12 @@ FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE ); +CREATE TABLE csrf_tokens ( + name text REFERENCES users(name) ON DELETE CASCADE, + token text, + end_date timestamp without time zone, + PRIMARY KEY(name) +); + + commit; Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Tue Aug 23 08:44:44 2011 @@ -16,6 +16,9 @@ from distutils.version import LooseVersion import trove, openid2rp from mini_pkg_resources import safe_name +# csrf modules +import hmac +from base64 import b64encode def enumerate(sequence): return [(i, sequence[i]) for i in range(len(sequence))] @@ -1778,6 +1781,44 @@ cursor = self.get_cursor() safe_execute(cursor, 'delete from cookies where cookie=%s', (cookie,)) + # CSRF Protection + + def get_token(self, username): + '''Return csrf current token for user.''' + cursor = self.get_cursor() + sql = '''select token from csrf_tokens where name=%s + and end_date > NOW()''' + safe_execute(cursor, sql, (username,)) + token = cursor.fetchall() + if not token: + return self.create_token(username) + return token[0][0] + + def create_token(self, username): + '''Create and return a new csrf token for user.''' + alphanum = string.ascii_letters + string.digits + # dependency on cookie existence + cursor = self.get_cursor() + safe_execute(cursor, 'select cookie from cookies where name=%s', + (username,)) + cookie = cursor.fetchall()[0][0] + # create random data + rand = [random.choice(alphanum) for i in range(12)] + rand.append(str(int(time.time()))) + rand.append(cookie) + random.shuffle(rand) + rand = hmac.new(''.join(random.choice(alphanum) for i in range(16)), + ''.join(rand),digestmod=hashlib.sha1).hexdigest() + rand = b64encode(rand) + + # we may have a current entry which is out of date, delete + safe_execute(cursor, 'delete from csrf_tokens where name=%s', (username,)) + sql = '''insert into csrf_tokens values(%s, %s, + NOW()+interval \'15 minutes\')''' + safe_execute(cursor, sql, (username, rand)) + + return rand + # OpenID def get_provider_session(self, provider): Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Aug 23 08:44:44 2011 @@ -337,6 +337,27 @@ template = PyPiPageTemplate(filename, template_dir) content = template(**context) + # dynamic insertion of CSRF token into FORMs + if '"POST"' in content and filename != 'pkg_edit.pt': + token = '<input type="hidden" name="CSRFToken" value="%s">' % ( + self.store.get_token(self.username),) + temp = content.split('\n') + edit = ((i, l) for i, l in enumerate(content.split('\n')) if + '"POST"' in l) + try: + for index, line in edit: + while not line.endswith('>'): + index += 1 + line = temp[index] + # count spaces to align entry nicely + spaces = len(line.lstrip()) - len(line) + temp[index] = "\n".join((line, ' ' * spaces + token)) + content = '\n'.join(temp) + except IndexError: + # this should not happen with correct HTML syntax + # the try is 'just in case someone does something stupid' + pass + self.handler.send_response(200, 'OK') if 'content-type' in options: self.handler.set_content_type(options['content-type']) @@ -512,6 +533,8 @@ if action in ('submit', ): if not self.authenticated: raise Unauthorised + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" if self.store.get_otk(self.username): raise Unauthorised, "Incomplete registration; check your email" @@ -910,6 +933,7 @@ if not self.authenticated: raise Unauthorised self.usercookie = self.store.create_cookie(self.username) + self.store.get_token(self.username) self.loggedin = 1 self.home() @@ -923,6 +947,8 @@ if not (self.store.has_role('Admin', package_name) or self.store.has_role('Owner', package_name)): raise Unauthorised + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" package = ''' <tr><th>Package Name:</th> <td><input type="text" readonly name="package_name" value="%s"></td> @@ -1517,6 +1543,7 @@ # Authenticated, but not logged in - auto-login self.loggedin = True self.usercookie = self.store.create_cookie(self.username) + token = self.store.get_token(self.username) # are we editing a specific entry? info = {} @@ -1588,6 +1615,9 @@ label = "Download URL" elif property == '_pypi_hidden': label = "Hidden" + elif property == 'CSRFToken': + field = '<input type="hidden" name="CSRFToken" value="%s">' % ( + token,) else: req = '' w('<tr><th %s>%s:</th><td>%s</td></tr>\n'%(req, label, @@ -1642,6 +1672,9 @@ raise FormError, \ "You must supply the PKG-INFO file" + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" + # get the data pkginfo = self.form['pkginfo'] if isinstance(pkginfo, FileUpload): @@ -1743,6 +1776,9 @@ self.validate_metadata(data) except ValueError, message: raise FormError, message + + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" name = data['name'] version = data['version'] @@ -1963,6 +1999,8 @@ if not self.authenticated: raise Unauthorised, \ "You must be identified to edit package information" + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" # vars name = self.form['name'] @@ -2092,6 +2130,8 @@ if not self.authenticated: raise Unauthorised, \ "You must be identified to edit package information" + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" # Verify protocol version if self.form.has_key('protocol_version'): @@ -2244,6 +2284,8 @@ if not self.authenticated: raise Unauthorised, \ "You must be identified to edit package information" + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" # figure the package name and version name = version = None @@ -2339,6 +2381,8 @@ ''' if not self.authenticated: raise Unauthorised, 'You must authenticate' + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" self.register_form() def register_form(self, openid_fields = (), username='', email='', openid=''): @@ -2524,6 +2568,8 @@ if "key" not in self.form: raise FormError, "missing key" + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" key = self.form['key'].splitlines() for line in key[1:]: @@ -2542,6 +2588,8 @@ raise Unauthorised if "id" not in self.form: raise FormError, "missing parameter" + if self.form['CSRFToken'] != self.store.get_token(self.username): + raise FormError, "Form Failure; reset form submission" try: id = int(self.form["id"]) except: @@ -2779,6 +2827,7 @@ self.username = user['name'] self.loggedin = self.authenticated = True self.usercookie = self.store.create_cookie(self.username) + self.store.get_token(self.username) return self.home() # Fill openid response fields into register form as hidden fields del qs[':action'] From python-checkins at python.org Tue Aug 23 08:45:02 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 08:45:02 +0200 (CEST) Subject: [Pypi-checkins] r959 - trunk/pypi/tools Message-ID: <3Rjj1p3cFRzNJZ@mail.python.org> Author: richard Date: Tue Aug 23 08:45:02 2011 New Revision: 959 Added: trunk/pypi/tools/sql-migrate-20110823-2.sql Log: addition of CSRF protection; thanks Capel Brunker Added: trunk/pypi/tools/sql-migrate-20110823-2.sql ============================================================================== --- (empty file) +++ trunk/pypi/tools/sql-migrate-20110823-2.sql Tue Aug 23 08:45:02 2011 @@ -0,0 +1,7 @@ +create table csrf_tokens ( + name text REFERENCES users(name) ON DELETE CASCADE, + token text, + end_date timestamp without time zone, + PRIMARY KEY(name) +); + From python-checkins at python.org Tue Aug 23 08:48:28 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 08:48:28 +0200 (CEST) Subject: [Pypi-checkins] r960 - trunk/pypi Message-ID: <3Rjj5m2HMwzNJZ@mail.python.org> Author: richard Date: Tue Aug 23 08:48:28 2011 New Revision: 960 Modified: trunk/pypi/webui.py Log: remove csrf token check from distutils register/upload command handlers Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Aug 23 08:48:28 2011 @@ -1776,9 +1776,6 @@ self.validate_metadata(data) except ValueError, message: raise FormError, message - - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" name = data['name'] version = data['version'] @@ -2130,8 +2127,6 @@ if not self.authenticated: raise Unauthorised, \ "You must be identified to edit package information" - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" # Verify protocol version if self.form.has_key('protocol_version'): From python-checkins at python.org Tue Aug 23 08:53:16 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 08:53:16 +0200 (CEST) Subject: [Pypi-checkins] r961 - trunk/pypi Message-ID: <3RjjCJ5ddqzNS4@mail.python.org> Author: richard Date: Tue Aug 23 08:53:16 2011 New Revision: 961 Modified: trunk/pypi/webui.py Log: not a form submission Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Aug 23 08:53:16 2011 @@ -947,8 +947,6 @@ if not (self.store.has_role('Admin', package_name) or self.store.has_role('Owner', package_name)): raise Unauthorised - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" package = ''' <tr><th>Package Name:</th> <td><input type="text" readonly name="package_name" value="%s"></td> From python-checkins at python.org Tue Aug 23 08:54:56 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 08:54:56 +0200 (CEST) Subject: [Pypi-checkins] r962 - trunk/pypi Message-ID: <3RjjFD1Y8szNRX@mail.python.org> Author: richard Date: Tue Aug 23 08:54:56 2011 New Revision: 962 Modified: trunk/pypi/webui.py Log: not a form submission Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Aug 23 08:54:56 2011 @@ -533,8 +533,6 @@ if action in ('submit', ): if not self.authenticated: raise Unauthorised - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" if self.store.get_otk(self.username): raise Unauthorised, "Incomplete registration; check your email" From python-checkins at python.org Tue Aug 23 09:10:43 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 09:10:43 +0200 (CEST) Subject: [Pypi-checkins] r963 - trunk/pypi Message-ID: <3RjjbR5yfbzNPQ@mail.python.org> Author: richard Date: Tue Aug 23 09:10:43 2011 New Revision: 963 Modified: trunk/pypi/store.py Log: handle missing cookie data Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Tue Aug 23 09:10:43 2011 @@ -1801,7 +1801,12 @@ cursor = self.get_cursor() safe_execute(cursor, 'select cookie from cookies where name=%s', (username,)) - cookie = cursor.fetchall()[0][0] + try: + cookie = cursor.fetchall()[0][0] + except IndexError: + # no cookie, make one + cookie = ''.join(random.choice(alphanum) for i in range(10)) + # create random data rand = [random.choice(alphanum) for i in range(12)] rand.append(str(int(time.time()))) From python-checkins at python.org Tue Aug 23 09:19:33 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 09:19:33 +0200 (CEST) Subject: [Pypi-checkins] r964 - trunk/pypi Message-ID: <3Rjjnd0L2TzNGZ@mail.python.org> Author: richard Date: Tue Aug 23 09:19:32 2011 New Revision: 964 Modified: trunk/pypi/webui.py Log: catch more pages that should not have the CSRF injected Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Aug 23 09:19:32 2011 @@ -338,7 +338,7 @@ content = template(**context) # dynamic insertion of CSRF token into FORMs - if '"POST"' in content and filename != 'pkg_edit.pt': + if '"POST"' in content and self.authenticated: token = '<input type="hidden" name="CSRFToken" value="%s">' % ( self.store.get_token(self.username),) temp = content.split('\n') From martin at v.loewis.de Tue Aug 23 12:00:40 2011 From: martin at v.loewis.de (=?ISO-8859-1?Q?=22Martin_v=2E_L=F6wis=22?=) Date: Tue, 23 Aug 2011 12:00:40 +0200 Subject: [Pypi-checkins] r959 - trunk/pypi/tools In-Reply-To: <3Rjj1p3cFRzNJZ@mail.python.org> References: <3Rjj1p3cFRzNJZ@mail.python.org> Message-ID: <4E537A48.2070801@v.loewis.de> > ============================================================================== > --- (empty file) > +++ trunk/pypi/tools/sql-migrate-20110823-2.sql Tue Aug 23 08:45:02 2011 > @@ -0,0 +1,7 @@ > +create table csrf_tokens ( > + name text REFERENCES users(name) ON DELETE CASCADE, > + token text, > + end_date timestamp without time zone, > + PRIMARY KEY(name) > +); > + Isn't the proper spelling of that create table csrf_tokens ( name text PRIMARY KEY REFERENCES users(name) ON DELETE CASCADE, token text, end_date timestamp without time zone ); Regards, Martin From python-checkins at python.org Tue Aug 23 12:56:26 2011 From: python-checkins at python.org (richard) Date: Tue, 23 Aug 2011 12:56:26 +0200 (CEST) Subject: [Pypi-checkins] r965 - trunk/pypi Message-ID: <3Rjpbt1L6SzMFQ@mail.python.org> Author: richard Date: Tue Aug 23 12:56:25 2011 New Revision: 965 Modified: trunk/pypi/webui.py Log: remove another couple of incorrect CSRF tests Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Aug 23 12:56:25 2011 @@ -2269,14 +2269,13 @@ # # Documentation Upload + # can't perform CSRF test as this might be invoked by a tool # def doc_upload(self): # make sure the user is identified if not self.authenticated: raise Unauthorised, \ "You must be identified to edit package information" - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" # figure the package name and version name = version = None @@ -2365,15 +2364,13 @@ self.wfile.write(c + '\n') # - # User handling code (registration, password changing + # User handling code (registration, password changing) # def user_form(self): ''' Make the user authenticate before viewing the "register" form. ''' if not self.authenticated: raise Unauthorised, 'You must authenticate' - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" self.register_form() def register_form(self, openid_fields = (), username='', email='', openid=''): From python-checkins at python.org Wed Aug 24 01:29:28 2011 From: python-checkins at python.org (richard) Date: Wed, 24 Aug 2011 01:29:28 +0200 (CEST) Subject: [Pypi-checkins] r966 - trunk/pypi Message-ID: <3Rk7Jm3LS3zNQx@mail.python.org> Author: richard Date: Wed Aug 24 01:29:28 2011 New Revision: 966 Modified: trunk/pypi/webui.py Log: fix a bunch of CSRF things: 1. handle the token not existing in the form submission at all (the most likely attack case) 2. assert the check in a few more places that were missed 3. explicitly note in a few places that it is not being checked because of command-line tools Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Wed Aug 24 01:29:28 2011 @@ -1008,6 +1008,8 @@ self.store.has_role('Owner', package_name)): raise Unauthorised + self.csrf_check() + # further vali:dation if role_name not in ('Owner', 'Maintainer'): raise FormError, 'role_name not Owner or Maintainer' @@ -1656,6 +1658,13 @@ title='Submitting package information', fields=content.getvalue().decode('utf8')) + def csrf_check(self): + '''Check that the required CSRF token is present in the form + submission. + ''' + if self.form.get('CSRFToken') != self.store.get_token(self.username): + self.FormError, "Form Failure; reset form submission" + def submit_pkg_info(self): ''' Handle the submission of distro metadata as a PKG-INFO file. ''' @@ -1668,8 +1677,7 @@ raise FormError, \ "You must supply the PKG-INFO file" - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" + self.csrf_check() # get the data pkginfo = self.form['pkginfo'] @@ -1937,6 +1945,8 @@ raise Unauthorised, \ "You must be identified to edit package information" + self.csrf_check() + name = self.form['name'] if self.form.has_key('submit_remove'): @@ -1992,8 +2002,8 @@ if not self.authenticated: raise Unauthorised, \ "You must be identified to edit package information" - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" + + self.csrf_check() # vars name = self.form['name'] @@ -2124,6 +2134,9 @@ raise Unauthorised, \ "You must be identified to edit package information" + # can't perform CSRF check as this is invoked by tools + #self.csrf_check() + # Verify protocol version if self.form.has_key('protocol_version'): protocol_version = self.form['protocol_version'] @@ -2269,7 +2282,6 @@ # # Documentation Upload - # can't perform CSRF test as this might be invoked by a tool # def doc_upload(self): # make sure the user is identified @@ -2277,6 +2289,9 @@ raise Unauthorised, \ "You must be identified to edit package information" + # can't perform CSRF check as this is invoked by tools + #self.csrf_check() + # figure the package name and version name = version = None if self.form.has_key('name'): @@ -2531,6 +2546,8 @@ 'indicated in the email.') % info['email'] else: + self.csrf_check() + # update details user = self.store.get_user(self.username) password = info.get('password', '').strip() @@ -2556,8 +2573,8 @@ if "key" not in self.form: raise FormError, "missing key" - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" + + self.csrf_check() key = self.form['key'].splitlines() for line in key[1:]: @@ -2574,10 +2591,12 @@ def delkey(self): if not self.authenticated: raise Unauthorised + if "id" not in self.form: raise FormError, "missing parameter" - if self.form['CSRFToken'] != self.store.get_token(self.username): - raise FormError, "Form Failure; reset form submission" + + self.csrf_check() + try: id = int(self.form["id"]) except: @@ -2647,7 +2666,9 @@ def delete_user(self): if not self.authenticated: raise Unauthorised + if self.form.has_key('submit_ok'): + self.csrf_check() # ok, do it self.store.delete_user(self.username) self.authenticated = self.loggedin = False @@ -2667,8 +2688,6 @@ return self.write_template('dialog.pt', message=message, title='Confirm account deletion', fields=fields) - - def send_email(self, recipient, message): ''' Send an administrative email to the recipient ''' @@ -2898,3 +2917,4 @@ stdout = p.communicate()[0] if p.returncode != 0: raise FormError, "Key processing failed. Please contact the administrator. Detail: "+stdout + From python-checkins at python.org Thu Aug 25 08:41:57 2011 From: python-checkins at python.org (richard) Date: Thu, 25 Aug 2011 08:41:57 +0200 (CEST) Subject: [Pypi-checkins] r967 - trunk/pypi/tools Message-ID: <3RkwsK1r02zMpq@mail.python.org> Author: richard Date: Thu Aug 25 08:41:50 2011 New Revision: 967 Removed: trunk/pypi/tools/create_all_fks.sql trunk/pypi/tools/drop_all_fks.sql Log: not needed Deleted: /trunk/pypi/tools/create_all_fks.sql ============================================================================== --- /trunk/pypi/tools/create_all_fks.sql Thu Aug 25 08:41:50 2011 +++ (empty file) @@ -1,55 +0,0 @@ -begin; --- OpenID tables - -ALTER TABLE openids ADD CONSTRAINT openids_name_fkey FOREIGN KEY (name) REFERENCES users (name) ON DELETE CASCADE; - -ALTER TABLE openid_stypes ADD CONSTRAINT openid_stypes_id_fkey FOREIGN KEY (id) REFERENCES openid_sessions ON DELETE CASCADE; - -ALTER TABLE cookies ADD CONSTRAINT cookies_name_fkey FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; - -ALTER TABLE sshkeys ADD CONSTRAINT sshkeys_name_fkey FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; - -ALTER TABLE rego_otk ADD CONSTRAINT rego_otk_name_fkey FOREIGN KEY (name) REFERENCES users ON DELETE CASCADE; - -ALTER TABLE journals ADD CONSTRAINT journals_submitted_by_fkey FOREIGN KEY (submitted_by) REFERENCES users ON DELETE CASCADE; - -ALTER TABLE cheesecake_subindices ADD CONSTRAINT cheesecake_subindices_main_index_id_fkey FOREIGN KEY (main_index_id) REFERENCES cheesecake_main_indices; - -ALTER TABLE releases ADD CONSTRAINT releases_name_fkey FOREIGN KEY (name) REFERENCES packages ON DELETE CASCADE; - -ALTER TABLE releases ADD CONSTRAINT releases_cheesecake_installability_id_fkey FOREIGN KEY (cheesecake_installability_id) REFERENCES cheesecake_main_indices; - -ALTER TABLE releases ADD CONSTRAINT releases_cheesecake_documentation_id_fkey FOREIGN KEY (cheesecake_documentation_id) REFERENCES cheesecake_main_indices; - -ALTER TABLE releases ADD CONSTRAINT releases_cheesecake_code_kwalitee_id_fkey FOREIGN KEY (cheesecake_code_kwalitee_id) REFERENCES cheesecake_main_indices; - -ALTER TABLE release_classifiers ADD CONSTRAINT release_classifiers_trove_id_fkey FOREIGN KEY (trove_id) REFERENCES trove_classifiers; - -ALTER TABLE release_classifiers ADD CONSTRAINT release_classifiers_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version); - -ALTER TABLE release_dependencies ADD CONSTRAINT release_dependencies_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE; - -ALTER TABLE release_files ADD CONSTRAINT release_files_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE; - -ALTER TABLE release_urls ADD CONSTRAINT release_urls_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE; - -ALTER TABLE description_urls ADD CONSTRAINT description_urls_name_fkey FOREIGN KEY (name, version) REFERENCES releases (name, version) ON UPDATE CASCADE; - -ALTER TABLE roles ADD CONSTRAINT roles_user_name_fkey FOREIGN KEY (user_name) REFERENCES users; -ALTER TABLE roles ADD CONSTRAINT roles_package_name_fkey FOREIGN KEY (package_name) REFERENCES packages ON UPDATE CASCADE; - -ALTER TABLE mirrors ADD CONSTRAINT mirrors_user_name_fkey FOREIGN KEY (user_name) REFERENCES users; - -ALTER TABLE ratings ADD CONSTRAINT ratings_user_name_fkey FOREIGN KEY (user_name) REFERENCES users ON DELETE CASCADE; - -ALTER TABLE ratings ADD CONSTRAINT ratings_name_fkey FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE; - -ALTER TABLE comments ADD CONSTRAINT comments_rating_fkey FOREIGN KEY (rating) REFERENCES ratings (id) ON DELETE CASCADE; - -ALTER TABLE comments ADD CONSTRAINT comments_user_name_fkey FOREIGN KEY (user_name) REFERENCES users ON DELETE CASCADE; - -ALTER TABLE comments ADD CONSTRAINT comments_in_reply_to_fkey FOREIGN KEY (in_reply_to) REFERENCES comments ON DELETE CASCADE; - -ALTER TABLE comments_journal ADD CONSTRAINT comments_journal_submitted_by_fkey FOREIGN KEY (submitted_by) REFERENCES users ON DELETE CASCADE; - -ALTER TABLE comments_journal ADD CONSTRAINT comments_journal_name_fkey FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE; Deleted: /trunk/pypi/tools/drop_all_fks.sql ============================================================================== --- /trunk/pypi/tools/drop_all_fks.sql Thu Aug 25 08:41:50 2011 +++ (empty file) @@ -1,35 +0,0 @@ -/* - -File Name : drop_all_fks.sql -Description: Drop all foreign keys for a PYPI database to allow pg_restore to - be run and clone the database -*/ - -ALTER TABLE openids DROP CONSTRAINT openids_name_fkey; -ALTER TABLE openid_stypes DROP CONSTRAINT openid_stypes_id_fkey; -ALTER TABLE cookies DROP CONSTRAINT cookies_name_fkey; -ALTER TABLE sshkeys DROP CONSTRAINT sshkeys_name_fkey; -ALTER TABLE rego_otk DROP CONSTRAINT rego_otk_name_fkey; -ALTER TABLE journals DROP CONSTRAINT journals_submitted_by_fkey; -ALTER TABLE cheesecake_subindices DROP CONSTRAINT cheesecake_subindices_main_index_id_fkey; -ALTER TABLE releases DROP CONSTRAINT releases_name_fkey; -ALTER TABLE releases DROP CONSTRAINT releases_cheesecake_installability_id_fkey; -ALTER TABLE releases DROP CONSTRAINT releases_cheesecake_documentation_id_fkey; -ALTER TABLE releases DROP CONSTRAINT releases_cheesecake_code_kwalitee_id_fkey; -ALTER TABLE release_classifiers DROP CONSTRAINT release_classifiers_trove_id_fkey; -ALTER TABLE release_classifiers DROP CONSTRAINT release_classifiers_name_fkey; -ALTER TABLE release_dependencies DROP CONSTRAINT release_dependencies_name_fkey; -ALTER TABLE release_files DROP CONSTRAINT release_files_name_fkey; -ALTER TABLE release_urls DROP CONSTRAINT release_urls_name_fkey; -ALTER TABLE description_urls DROP CONSTRAINT description_urls_name_fkey; -ALTER TABLE roles DROP CONSTRAINT roles_user_name_fkey; -ALTER TABLE roles DROP CONSTRAINT roles_package_name_fkey; -ALTER TABLE mirrors DROP CONSTRAINT mirrors_user_name_fkey; -ALTER TABLE ratings DROP CONSTRAINT ratings_user_name_fkey; -ALTER TABLE ratings DROP CONSTRAINT ratings_name_fkey; -ALTER TABLE comments DROP CONSTRAINT comments_rating_fkey; -ALTER TABLE comments DROP CONSTRAINT comments_user_name_fkey; -ALTER TABLE comments DROP CONSTRAINT comments_in_reply_to_fkey; -ALTER TABLE comments_journal DROP CONSTRAINT comments_journal_submitted_by_fkey; -ALTER TABLE comments_journal DROP CONSTRAINT comments_journal_name_fkey; -
Journal