[Python-checkins] r83211 - in python/branches/release27-maint: Lib/robotparser.py Lib/test/test_robotparser.py

[senthil.kumaran]

Author: senthil.kumaran
Date: Wed Jul 28 18:35:35 2010
New Revision: 83211

Log:
Merged revisions 83209 via svnmerge from 
svn+ssh://pythondev@svn.python.org/python/branches/py3k

........
  r83209 | senthil.kumaran | 2010-07-28 21:57:56 +0530 (Wed, 28 Jul 2010) | 3 lines
  
  Fix Issue6325 - robotparse to honor urls with query strings.
........


Modified:
   python/branches/release27-maint/   (props changed)
   python/branches/release27-maint/Lib/robotparser.py
   python/branches/release27-maint/Lib/test/test_robotparser.py

Modified: python/branches/release27-maint/Lib/robotparser.py
==============================================================================
--- python/branches/release27-maint/Lib/robotparser.py	(original)
+++ python/branches/release27-maint/Lib/robotparser.py	Wed Jul 28 18:35:35 2010
@@ -131,7 +131,12 @@
             return True
         # search for given user agent matches
         # the first match counts
-        url = urllib.quote(urlparse.urlparse(urllib.unquote(url))[2]) or "/"
+        parsed_url = urlparse.urlparse(urllib.unquote(url))
+        url = urlparse.urlunparse(('', '', parsed_url.path,
+            parsed_url.params, parsed_url.query, parsed_url.fragment))
+        url = urllib.quote(url)
+        if not url:
+            url = "/"
         for entry in self.entries:
             if entry.applies_to(useragent):
                 return entry.allowance(url)

Modified: python/branches/release27-maint/Lib/test/test_robotparser.py
==============================================================================
--- python/branches/release27-maint/Lib/test/test_robotparser.py	(original)
+++ python/branches/release27-maint/Lib/test/test_robotparser.py	Wed Jul 28 18:35:35 2010
@@ -202,6 +202,17 @@
 RobotTest(13, doc, good, bad, agent="googlebot")
 
 
+# 14. For issue #6325 (query string support)
+doc = """
+User-agent: *
+Disallow: /some/path?name=value
+"""
+
+good = ['/some/path']
+bad = ['/some/path?name=value']
+
+RobotTest(14, doc, good, bad)
+
 
 class NetworkTestCase(unittest.TestCase):