[Python-checkins] r60276 - python/trunk/Lib/test/test_urllib2net.py
neal.norwitz
python-checkins at python.org
Fri Jan 25 07:37:24 CET 2008
Author: neal.norwitz
Date: Fri Jan 25 07:37:23 2008
New Revision: 60276
Modified:
python/trunk/Lib/test/test_urllib2net.py
Log:
Make the test more robust by trying to reconnect up to 3 times
in case there were transient failures. This will hopefully silence
the buildbots for this test. As we find other tests that have a problem,
we can fix with a similar strategy assuming it is successful. It worked
on my box in a loop for 10+ runs where it would have an exception otherwise.
Modified: python/trunk/Lib/test/test_urllib2net.py
==============================================================================
--- python/trunk/Lib/test/test_urllib2net.py (original)
+++ python/trunk/Lib/test/test_urllib2net.py Fri Jan 25 07:37:23 2008
@@ -10,6 +10,20 @@
import os
import mimetools
+
+def _urlopen_with_retry(host, *args, **kwargs):
+ # Connecting to remote hosts is flaky. Make it more robust
+ # by retrying the connection several times.
+ for i in range(3):
+ try:
+ return urllib2.urlopen(host, *args, **kwargs)
+ except urllib2.URLError, last_exc:
+ continue
+ except:
+ raise
+ raise last_exc
+
+
class URLTimeoutTest(unittest.TestCase):
TIMEOUT = 10.0
@@ -21,7 +35,7 @@
socket.setdefaulttimeout(None)
def testURLread(self):
- f = urllib2.urlopen("http://www.python.org/")
+ f = _urlopen_with_retry("http://www.python.org/")
x = f.read()
@@ -42,7 +56,7 @@
#
# # failure
# try:
-# urllib2.urlopen(test_url)
+# _urlopen_with_retry(test_url)
# except urllib2.HTTPError, exc:
# self.assertEqual(exc.code, 401)
# else:
@@ -54,7 +68,7 @@
# test_user, test_password)
# opener = urllib2.build_opener(auth_handler)
# f = opener.open('http://localhost/')
-# response = urllib2.urlopen("http://www.python.org/")
+# response = _urlopen_with_retry("http://www.python.org/")
#
# # The 'userinfo' URL component is deprecated by RFC 3986 for security
# # reasons, let's not implement it! (it's already implemented for proxy
@@ -73,7 +87,7 @@
# underlying socket
# delve deep into response to fetch socket._socketobject
- response = urllib2.urlopen("http://www.python.org/")
+ response = _urlopen_with_retry("http://www.python.org/")
abused_fileobject = response.fp
self.assert_(abused_fileobject.__class__ is socket._fileobject)
httpresponse = abused_fileobject._sock
@@ -102,7 +116,7 @@
def test_basic(self):
# Simple test expected to pass.
- open_url = urllib2.urlopen("http://www.python.org/")
+ open_url = _urlopen_with_retry("http://www.python.org/")
for attr in ("read", "close", "info", "geturl"):
self.assert_(hasattr(open_url, attr), "object returned from "
"urlopen lacks the %s attribute" % attr)
@@ -113,7 +127,7 @@
def test_info(self):
# Test 'info'.
- open_url = urllib2.urlopen("http://www.python.org/")
+ open_url = _urlopen_with_retry("http://www.python.org/")
try:
info_obj = open_url.info()
finally:
@@ -126,7 +140,7 @@
def test_geturl(self):
# Make sure same URL as opened is returned by geturl.
URL = "http://www.python.org/"
- open_url = urllib2.urlopen(URL)
+ open_url = _urlopen_with_retry(URL)
try:
gotten_url = open_url.geturl()
finally:
@@ -157,7 +171,7 @@
def test_range (self):
req = urllib2.Request("http://www.python.org",
headers={'Range': 'bytes=20-39'})
- result = urllib2.urlopen(req)
+ result = _urlopen_with_retry(req)
data = result.read()
self.assertEqual(len(data), 20)
@@ -184,7 +198,7 @@
'file:'+sanepathname2url(os.path.abspath(TESTFN)),
('file:///nonsensename/etc/passwd', None, urllib2.URLError),
]
- self._test_urls(urls, self._extra_handlers())
+ self._test_urls(urls, self._extra_handlers(), urllib2.urlopen)
finally:
os.remove(TESTFN)
@@ -226,7 +240,7 @@
## self._test_urls(urls, self._extra_handlers()+[bauth, dauth])
- def _test_urls(self, urls, handlers):
+ def _test_urls(self, urls, handlers, urlopen=_urlopen_with_retry):
import socket
import time
import logging
@@ -241,7 +255,7 @@
req = expected_err = None
debug(url)
try:
- f = urllib2.urlopen(url, req)
+ f = urlopen(url, req)
except EnvironmentError, err:
debug(err)
if expected_err:
@@ -267,45 +281,47 @@
class TimeoutTest(unittest.TestCase):
def test_http_basic(self):
- u = urllib2.urlopen("http://www.python.org")
+ u = _urlopen_with_retry("http://www.python.org")
self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None)
def test_http_NoneWithdefault(self):
prev = socket.getdefaulttimeout()
socket.setdefaulttimeout(60)
try:
- u = urllib2.urlopen("http://www.python.org", timeout=None)
+ u = _urlopen_with_retry("http://www.python.org", timeout=None)
self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60)
finally:
socket.setdefaulttimeout(prev)
def test_http_Value(self):
- u = urllib2.urlopen("http://www.python.org", timeout=120)
+ u = _urlopen_with_retry("http://www.python.org", timeout=120)
self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120)
def test_http_NoneNodefault(self):
- u = urllib2.urlopen("http://www.python.org", timeout=None)
+ u = _urlopen_with_retry("http://www.python.org", timeout=None)
self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None)
+ FTP_HOST = "ftp://ftp.mirror.nl/pub/mirror/gnu/"
+
def test_ftp_basic(self):
- u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/")
+ u = _urlopen_with_retry(self.FTP_HOST)
self.assertTrue(u.fp.fp._sock.gettimeout() is None)
def test_ftp_NoneWithdefault(self):
prev = socket.getdefaulttimeout()
socket.setdefaulttimeout(60)
try:
- u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=None)
+ u = _urlopen_with_retry(self.FTP_HOST, timeout=None)
self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
finally:
socket.setdefaulttimeout(prev)
def test_ftp_NoneNodefault(self):
- u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=None)
+ u = _urlopen_with_retry(self.FTP_HOST, timeout=None)
self.assertTrue(u.fp.fp._sock.gettimeout() is None)
def test_ftp_Value(self):
- u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=60)
+ u = _urlopen_with_retry(self.FTP_HOST, timeout=60)
self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
More information about the Python-checkins
mailing list