[Python-checkins] [3.7] bpo-32861: urllib.robotparser fix incomplete __str__ methods. (GH-5711) (GH-6795)
Serhiy Storchaka
webhook-mailer at python.org
Mon May 14 14:14:33 EDT 2018
https://github.com/python/cpython/commit/c3fa1f2b93fa4bf96a8aadc74ee196384cefa31e
commit: c3fa1f2b93fa4bf96a8aadc74ee196384cefa31e
branch: 3.7
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Serhiy Storchaka <storchaka at gmail.com>
date: 2018-05-14T21:14:30+03:00
summary:
[3.7] bpo-32861: urllib.robotparser fix incomplete __str__ methods. (GH-5711) (GH-6795)
The urllib.robotparser's __str__ representation now includes wildcard
entries and the "Crawl-delay" and "Request-rate" fields.
(cherry picked from commit bd08a0af2d88c590ede762102bd42da3437e9980)
Co-authored-by: Michael Lazar <lazar.michael22 at gmail.com>
files:
A Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst
M Lib/test/test_robotparser.py
M Lib/urllib/robotparser.py
diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py
index 75198b70ad4f..140636590aa8 100644
--- a/Lib/test/test_robotparser.py
+++ b/Lib/test/test_robotparser.py
@@ -246,6 +246,33 @@ class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase):
bad = ['/cyberworld/map/index.html']
+class StringFormattingTest(BaseRobotTest, unittest.TestCase):
+ robots_txt = """\
+User-agent: *
+Crawl-delay: 1
+Request-rate: 3/15
+Disallow: /cyberworld/map/ # This is an infinite virtual URL space
+
+# Cybermapper knows where to go.
+User-agent: cybermapper
+Disallow: /some/path
+ """
+
+ expected_output = """\
+User-agent: cybermapper
+Disallow: /some/path
+
+User-agent: *
+Crawl-delay: 1
+Request-rate: 3/15
+Disallow: /cyberworld/map/
+
+"""
+
+ def test_string_formatting(self):
+ self.assertEqual(str(self.parser), self.expected_output)
+
+
class RobotHandler(BaseHTTPRequestHandler):
def do_GET(self):
diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py
index daac29c68dc3..883ef249210e 100644
--- a/Lib/urllib/robotparser.py
+++ b/Lib/urllib/robotparser.py
@@ -190,7 +190,10 @@ def request_rate(self, useragent):
return self.default_entry.req_rate
def __str__(self):
- return ''.join([str(entry) + "\n" for entry in self.entries])
+ entries = self.entries
+ if self.default_entry is not None:
+ entries = entries + [self.default_entry]
+ return '\n'.join(map(str, entries)) + '\n'
class RuleLine:
@@ -222,10 +225,15 @@ def __init__(self):
def __str__(self):
ret = []
for agent in self.useragents:
- ret.extend(["User-agent: ", agent, "\n"])
- for line in self.rulelines:
- ret.extend([str(line), "\n"])
- return ''.join(ret)
+ ret.append(f"User-agent: {agent}")
+ if self.delay is not None:
+ ret.append(f"Crawl-delay: {self.delay}")
+ if self.req_rate is not None:
+ rate = self.req_rate
+ ret.append(f"Request-rate: {rate.requests}/{rate.seconds}")
+ ret.extend(map(str, self.rulelines))
+ ret.append('') # for compatibility
+ return '\n'.join(ret)
def applies_to(self, useragent):
"""check if this entry applies to the specified agent"""
diff --git a/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst
new file mode 100644
index 000000000000..13defbb03cff
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-04-02-20-44-54.bpo-32861.HeBjzN.rst
@@ -0,0 +1,3 @@
+The urllib.robotparser's ``__str__`` representation now includes wildcard
+entries and the "Crawl-delay" and "Request-rate" fields. Patch by
+Michael Lazar.
More information about the Python-checkins
mailing list