http://hg.python.org/cpython/rev/e3e5b6f03f79 changeset: 73257:e3e5b6f03f79 parent: 73255:c14e00680a99 parent: 73256:2ca415cbf2ac user: Senthil Kumaran <senthil@uthcode.com> date: Tue Nov 01 01:39:49 2011 +0800 summary: merge from 3.2 - Fix issue 10817 - Fix urlretrieve function to raise ContentTooShortError even when reporthook is None. Patch by Jyrki Pulliainen. files: Lib/test/test_urllib.py | 109 +++++++++++++++++++++------ Lib/urllib/request.py | 4 +- Misc/NEWS | 3 + 3 files changed, 87 insertions(+), 29 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -36,6 +36,44 @@ else: return opener.open(url, data) + +class FakeHTTPMixin(object): + def fakehttp(self, fakedata): + class FakeSocket(io.BytesIO): + io_refs = 1 + + def sendall(self, str): + pass + + def makefile(self, *args, **kwds): + self.io_refs += 1 + return self + + def read(self, amt=None): + if self.closed: + return b"" + return io.BytesIO.read(self, amt) + + def readline(self, length=None): + if self.closed: + return b"" + return io.BytesIO.readline(self, length) + + def close(self): + self.io_refs -= 1 + if self.io_refs == 0: + io.BytesIO.close(self) + + class FakeHTTPConnection(http.client.HTTPConnection): + def connect(self): + self.sock = FakeSocket(fakedata) + self._connection_class = http.client.HTTPConnection + http.client.HTTPConnection = FakeHTTPConnection + + def unfakehttp(self): + http.client.HTTPConnection = self._connection_class + + class urlopen_FileTests(unittest.TestCase): """Test urlopen() opening a temporary file. @@ -139,35 +177,9 @@ self.env.set('NO_PROXY', 'localhost, anotherdomain.com, newdomain.com') self.assertTrue(urllib.request.proxy_bypass_environment('anotherdomain.com')) -class urlopen_HttpTests(unittest.TestCase): +class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin): """Test urlopen() opening a fake http connection.""" - def fakehttp(self, fakedata): - class FakeSocket(io.BytesIO): - io_refs = 1 - def sendall(self, str): pass - def makefile(self, *args, **kwds): - self.io_refs += 1 - return self - def read(self, amt=None): - if self.closed: return b"" - return io.BytesIO.read(self, amt) - def readline(self, length=None): - if self.closed: return b"" - return io.BytesIO.readline(self, length) - def close(self): - self.io_refs -= 1 - if self.io_refs == 0: - io.BytesIO.close(self) - class FakeHTTPConnection(http.client.HTTPConnection): - def connect(self): - self.sock = FakeSocket(fakedata) - self._connection_class = http.client.HTTPConnection - http.client.HTTPConnection = FakeHTTPConnection - - def unfakehttp(self): - http.client.HTTPConnection = self._connection_class - def check_read(self, ver): self.fakehttp(b"HTTP/" + ver + b" 200 OK\r\n\r\nHello!") try: @@ -394,6 +406,48 @@ self.assertEqual(report[0][1], 8192) self.assertEqual(report[0][2], 8193) + +class urlretrieve_HttpTests(unittest.TestCase, FakeHTTPMixin): + """Test urllib.urlretrieve() using fake http connections""" + + def test_short_content_raises_ContentTooShortError(self): + self.fakehttp(b'''HTTP/1.1 200 OK +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Length: 100 +Content-Type: text/html; charset=iso-8859-1 + +FF +''') + + def _reporthook(par1, par2, par3): + pass + + with self.assertRaises(urllib.error.ContentTooShortError): + try: + urllib.request.urlretrieve('http://example.com/', + reporthook=_reporthook) + finally: + self.unfakehttp() + + def test_short_content_raises_ContentTooShortError_without_reporthook(self): + self.fakehttp(b'''HTTP/1.1 200 OK +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Connection: close +Content-Length: 100 +Content-Type: text/html; charset=iso-8859-1 + +FF +''') + with self.assertRaises(urllib.error.ContentTooShortError): + try: + urllib.request.urlretrieve('http://example.com/') + finally: + self.unfakehttp() + + class QuotingTests(unittest.TestCase): """Tests for urllib.quote() and urllib.quote_plus() @@ -1186,6 +1240,7 @@ urlopen_FileTests, urlopen_HttpTests, urlretrieve_FileTests, + urlretrieve_HttpTests, ProxyTests, QuotingTests, UnquotingTests, diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1605,9 +1605,9 @@ size = -1 read = 0 blocknum = 0 + if "content-length" in headers: + size = int(headers["Content-Length"]) if reporthook: - if "content-length" in headers: - size = int(headers["Content-Length"]) reporthook(blocknum, bs, size) while 1: block = fp.read(bs) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -347,6 +347,9 @@ Library ------- +- Issue 10817: Fix urlretrieve function to raise ContentTooShortError even + when reporthook is None. Patch by Jyrki Pulliainen. + - Issue 13296: Fix IDLE to clear compile __future__ flags on shell restart. (Patch by Roger Serwy) -- Repository URL: http://hg.python.org/cpython