Mailman 3 March 2011 - Python-checkins

cpython (merge 3.2 -> default): Merge Issue 11662 from 3.2 branch.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/a778b963eae3 changeset: 69049:a778b963eae3 parent: 68901:4f1cd92fe835 parent: 69048:96a6c128822b user: guido(a)google.com date: Tue Mar 29 12:02:49 2011 -0700 summary: Merge Issue 11662 from 3.2 branch. files: Doc/library/urllib.request.rst | 4 +++ Lib/test/test_urllib.py | 16 +++++++++++++ Lib/test/test_urllib2.py | 24 +++++++++++++++++++ Lib/urllib/request.py | 27 ++++++++++++++++++++++ Misc/NEWS | 3 ++ 5 files changed, 74 insertions(+), 0 deletions(-) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -650,6 +650,10 @@ is the case, :exc:`HTTPError` is raised. See :rfc:`2616` for details of the precise meanings of the various redirection codes. + An :class:`HTTPError` exception raised as a security consideration if the + HTTPRedirectHandler is presented with a redirected url which is not an HTTP, + HTTPS or FTP url. + .. method:: HTTPRedirectHandler.redirect_request(req, fp, code, msg, hdrs, newurl) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -2,6 +2,7 @@ import urllib.parse import urllib.request +import urllib.error import http.client import email.message import io @@ -206,6 +207,21 @@ finally: self.unfakehttp() + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp(b'''HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file://guidocomputer.athome.com:/python/license +Connection: close +Content-Type: text/html; charset=iso-8859-1 +''') + try: + self.assertRaises(urllib.error.HTTPError, urlopen, + "http://python.org/") + finally: + self.unfakehttp() + def test_empty_socket(self): # urlopen() raises IOError if the underlying socket does not send any # data. (#1680230) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -10,6 +10,7 @@ # The proxy bypass method imported below has logic specific to the OSX # proxy config data structure but is testable on all platforms. from urllib.request import Request, OpenerDirector, _proxy_bypass_macosx_sysconf +import urllib.error # XXX # Request @@ -1031,6 +1032,29 @@ self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_redirections) + + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http','https','ftp'] + invalid_schemes = ['file','imap','ldap'] + schemeless_url = "example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib.error.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from http.cookiejar import CookieJar diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -545,6 +545,17 @@ # fix a possible malformed URL urlparts = urlparse(newurl) + + # For security reasons we don't allow redirection to anything other + # than http, https or ftp. + + if not urlparts.scheme in ('http', 'https', 'ftp'): + raise HTTPError(newurl, code, + msg + + " - Redirection to url '%s' is not allowed" % + newurl, + headers, fp) + if not urlparts.path: urlparts = list(urlparts) urlparts[2] = "/" @@ -1903,8 +1914,24 @@ return void = fp.read() fp.close() + # In case the server sent a relative URL, join with original: newurl = urljoin(self.type + ":" + url, newurl) + + urlparts = urlparse(newurl) + + # For security reasons, we don't allow redirection to anything other + # than http, https and ftp. + + # We are using newer HTTPError with older redirect_internal method + # This older method will get deprecated in 3.3 + + if not urlparts.scheme in ('http', 'https', 'ftp'): + raise HTTPError(newurl, errcode, + errmsg + + " Redirection to url '%s' is not allowed." % newurl, + headers, fp) + return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -84,6 +84,9 @@ Library ------- +- Issue #11662: Make urllib and urllib2 ignore redirections if the + scheme is not HTTP, HTTPS or FTP (CVE-2011-1521). + - Issue #6811: Allow importlib to change a code object's co_filename attribute to match the path to where the source code currently is, not where the code object originally came from. -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 3.1 -> 3.2): Merge Issue 11662 from 3.1 branch.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/96a6c128822b changeset: 69048:96a6c128822b branch: 3.2 parent: 68899:b801d55a9979 parent: 69047:5937d2119a20 user: guido(a)google.com date: Tue Mar 29 11:51:26 2011 -0700 summary: Merge Issue 11662 from 3.1 branch. files: Doc/library/urllib.request.rst | 4 +++ Lib/test/test_urllib.py | 16 +++++++++++++ Lib/test/test_urllib2.py | 24 +++++++++++++++++++ Lib/urllib/request.py | 27 ++++++++++++++++++++++ Misc/NEWS | 3 ++ Objects/typeslots.inc | 2 +- 6 files changed, 75 insertions(+), 1 deletions(-) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -650,6 +650,10 @@ is the case, :exc:`HTTPError` is raised. See :rfc:`2616` for details of the precise meanings of the various redirection codes. + An :class:`HTTPError` exception raised as a security consideration if the + HTTPRedirectHandler is presented with a redirected url which is not an HTTP, + HTTPS or FTP url. + .. method:: HTTPRedirectHandler.redirect_request(req, fp, code, msg, hdrs, newurl) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -2,6 +2,7 @@ import urllib.parse import urllib.request +import urllib.error import http.client import email.message import io @@ -206,6 +207,21 @@ finally: self.unfakehttp() + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp(b'''HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file://guidocomputer.athome.com:/python/license +Connection: close +Content-Type: text/html; charset=iso-8859-1 +''') + try: + self.assertRaises(urllib.error.HTTPError, urlopen, + "http://python.org/") + finally: + self.unfakehttp() + def test_empty_socket(self): # urlopen() raises IOError if the underlying socket does not send any # data. (#1680230) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -10,6 +10,7 @@ # The proxy bypass method imported below has logic specific to the OSX # proxy config data structure but is testable on all platforms. from urllib.request import Request, OpenerDirector, _proxy_bypass_macosx_sysconf +import urllib.error # XXX # Request @@ -1031,6 +1032,29 @@ self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_redirections) + + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http','https','ftp'] + invalid_schemes = ['file','imap','ldap'] + schemeless_url = "example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib.error.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from http.cookiejar import CookieJar diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -545,6 +545,17 @@ # fix a possible malformed URL urlparts = urlparse(newurl) + + # For security reasons we don't allow redirection to anything other + # than http, https or ftp. + + if not urlparts.scheme in ('http', 'https', 'ftp'): + raise HTTPError(newurl, code, + msg + + " - Redirection to url '%s' is not allowed" % + newurl, + headers, fp) + if not urlparts.path: urlparts = list(urlparts) urlparts[2] = "/" @@ -1903,8 +1914,24 @@ return void = fp.read() fp.close() + # In case the server sent a relative URL, join with original: newurl = urljoin(self.type + ":" + url, newurl) + + urlparts = urlparse(newurl) + + # For security reasons, we don't allow redirection to anything other + # than http, https and ftp. + + # We are using newer HTTPError with older redirect_internal method + # This older method will get deprecated in 3.3 + + if not urlparts.scheme in ('http', 'https', 'ftp'): + raise HTTPError(newurl, errcode, + errmsg + + " Redirection to url '%s' is not allowed." % newurl, + headers, fp) + return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -49,6 +49,9 @@ Library ------- +- Issue #11662: Make urllib and urllib2 ignore redirections if the + scheme is not HTTP, HTTPS or FTP (CVE-2011-1521). + - Issue #11628: cmp_to_key generated class should use __slots__ - Issue #5537: Fix time2isoz() and time2netscape() functions of diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -1,4 +1,4 @@ -/* Generated by typeslots.py $Revision: 87806 $ */ +/* Generated by typeslots.py $Revision$ */ 0, 0, offsetof(PyHeapTypeObject, as_mapping.mp_ass_subscript), -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.1): Issue 11662: Fix vulnerability in urllib/urllib2.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/5937d2119a20 changeset: 69047:5937d2119a20 branch: 3.1 parent: 68898:37ba11d806c5 user: guido(a)google.com date: Tue Mar 29 11:41:02 2011 -0700 summary: Issue 11662: Fix vulnerability in urllib/urllib2. (This version is a cleaned-up backport of a fix by Senthil Kumaran.) files: Doc/library/urllib.request.rst | 4 +++ Lib/test/test_urllib.py | 16 +++++++++++++ Lib/test/test_urllib2.py | 24 +++++++++++++++++++ Lib/urllib/request.py | 27 ++++++++++++++++++++++ Misc/NEWS | 3 ++ 5 files changed, 74 insertions(+), 0 deletions(-) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -783,6 +783,10 @@ is the case, :exc:`HTTPError` is raised. See :rfc:`2616` for details of the precise meanings of the various redirection codes. + An :class:`HTTPError` exception raised as a security consideration if the + HTTPRedirectHandler is presented with a redirected url which is not an HTTP, + HTTPS or FTP url. + .. method:: HTTPRedirectHandler.redirect_request(req, fp, code, msg, hdrs, newurl) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -2,6 +2,7 @@ import urllib.parse import urllib.request +import urllib.error import http.client import email.message import io @@ -183,6 +184,21 @@ finally: self.unfakehttp() + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp(b'''HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file://guidocomputer.athome.com:/python/license +Connection: close +Content-Type: text/html; charset=iso-8859-1 +''') + try: + self.assertRaises(urllib.error.HTTPError, urlopen, + "http://python.org/") + finally: + self.unfakehttp() + def test_empty_socket(self): # urlopen() raises IOError if the underlying socket does not send any # data. (#1680230) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -9,6 +9,7 @@ # The proxy bypass method imported below has logic specific to the OSX # proxy config data structure but is testable on all platforms. from urllib.request import Request, OpenerDirector, _proxy_bypass_macosx_sysconf +import urllib.error # XXX # Request @@ -985,6 +986,29 @@ self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_redirections) + + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http','https','ftp'] + invalid_schemes = ['file','imap','ldap'] + schemeless_url = "example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib.error.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from http.cookiejar import CookieJar diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -528,6 +528,17 @@ # fix a possible malformed URL urlparts = urlparse(newurl) + + # For security reasons we don't allow redirection to anything other + # than http, https or ftp. + + if not urlparts.scheme in ('http', 'https', 'ftp'): + raise HTTPError(newurl, code, + msg + + " - Redirection to url '%s' is not allowed" % + newurl, + headers, fp) + if not urlparts.path: urlparts = list(urlparts) urlparts[2] = "/" @@ -1864,8 +1875,24 @@ return void = fp.read() fp.close() + # In case the server sent a relative URL, join with original: newurl = urljoin(self.type + ":" + url, newurl) + + urlparts = urlparse(newurl) + + # For security reasons, we don't allow redirection to anything other + # than http, https and ftp. + + # We are using newer HTTPError with older redirect_internal method + # This older method will get deprecated in 3.3 + + if not urlparts.scheme in ('http', 'https', 'ftp'): + raise HTTPError(newurl, errcode, + errmsg + + " Redirection to url '%s' is not allowed." % newurl, + headers, fp) + return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -44,6 +44,9 @@ Library ------- +- Issue #11662: Make urllib and urllib2 ignore redirections if the + scheme is not HTTP, HTTPS or FTP (CVE-2011-1521). + - Issue #5537: Fix time2isoz() and time2netscape() functions of httplib.cookiejar for expiration year greater than 2038 on 32-bit systems. -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 2.6 -> 2.7): Merge urllib/urllib2 security fix from 2.6 branch.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/163dc7be1afd changeset: 69046:163dc7be1afd branch: 2.7 parent: 68856:24179f82b7de parent: 69045:c15ab33017ff user: guido(a)google.com date: Tue Mar 29 11:14:01 2011 -0700 summary: Merge urllib/urllib2 security fix from 2.6 branch. files: Lib/test/test_urllib.py | 14 ++++++++++++++ Lib/test/test_urllib2.py | 22 ++++++++++++++++++++++ Lib/urllib.py | 12 ++++++++++++ Lib/urllib2.py | 11 +++++++++++ Misc/NEWS | 2 ++ 5 files changed, 61 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -161,6 +161,20 @@ finally: self.unfakehttp() + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp("""HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file:README +Connection: close +Content-Type: text/html; charset=iso-8859-1 +""") + try: + self.assertRaises(IOError, urllib.urlopen, "http://python.org/") + finally: + self.unfakehttp() + def test_empty_socket(self): # urlopen() raises IOError if the underlying socket does not send any # data. (#1680230) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -969,6 +969,28 @@ self.assertEqual(count, urllib2.HTTPRedirectHandler.max_redirections) + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http', 'https', 'ftp'] + invalid_schemes = ['file', 'imap', 'ldap'] + schemeless_url = "example.com/b.html" + h = urllib2.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib2.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from cookielib import CookieJar diff --git a/Lib/urllib.py b/Lib/urllib.py --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -644,6 +644,18 @@ fp.close() # In case the server sent a relative URL, join with original: newurl = basejoin(self.type + ":" + url, newurl) + + # For security reasons we do not allow redirects to protocols + # other than HTTP, HTTPS or FTP. + newurl_lower = newurl.lower() + if not (newurl_lower.startswith('http://') or + newurl_lower.startswith('https://') or + newurl_lower.startswith('ftp://')): + raise IOError('redirect error', errcode, + errmsg + " - Redirection to url '%s' is not allowed" % + newurl, + headers) + return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Lib/urllib2.py b/Lib/urllib2.py --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -578,6 +578,17 @@ newurl = urlparse.urljoin(req.get_full_url(), newurl) + # For security reasons we do not allow redirects to protocols + # other than HTTP, HTTPS or FTP. + newurl_lower = newurl.lower() + if not (newurl_lower.startswith('http://') or + newurl_lower.startswith('https://') or + newurl_lower.startswith('ftp://')): + raise HTTPError(newurl, code, + msg + " - Redirection to url '%s' is not allowed" % + newurl, + headers, fp) + # XXX Probably want to forget about the state of the current # request, although that might interact poorly with other # handlers that also use handler-specific request attributes diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -42,6 +42,8 @@ Library ------- +- Issue #11662: Make urllib and urllib2 ignore redirections if the + scheme is not HTTP, HTTPS or FTP (CVE-2011-1521). - Issue #10042: Fixed the total_ordering decorator to handle cross-type comparisons that could lead to infinite recursion. -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 2.5 -> 2.6): Merge urllib/urllib2 security fix from 2.5 branch.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/c15ab33017ff changeset: 69045:c15ab33017ff branch: 2.6 parent: 68802:b99c94261225 parent: 69044:92293101839c user: guido(a)google.com date: Tue Mar 29 10:48:23 2011 -0700 summary: Merge urllib/urllib2 security fix from 2.5 branch. files: Lib/test/test_urllib.py | 14 ++++++++++++++ Lib/test/test_urllib2.py | 22 ++++++++++++++++++++++ Lib/urllib.py | 12 ++++++++++++ Lib/urllib2.py | 11 +++++++++++ Misc/NEWS | 2 ++ 5 files changed, 61 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -162,6 +162,20 @@ finally: self.unfakehttp() + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp("""HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file:README +Connection: close +Content-Type: text/html; charset=iso-8859-1 +""") + try: + self.assertRaises(IOError, urllib.urlopen, "http://python.org/") + finally: + self.unfakehttp() + def test_empty_socket(self): # urlopen() raises IOError if the underlying socket does not send any # data. (#1680230) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -942,6 +942,28 @@ self.assertEqual(count, urllib2.HTTPRedirectHandler.max_redirections) + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http', 'https', 'ftp'] + invalid_schemes = ['file', 'imap', 'ldap'] + schemeless_url = "example.com/b.html" + h = urllib2.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib2.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from cookielib import CookieJar diff --git a/Lib/urllib.py b/Lib/urllib.py --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -652,6 +652,18 @@ fp.close() # In case the server sent a relative URL, join with original: newurl = basejoin(self.type + ":" + url, newurl) + + # For security reasons we do not allow redirects to protocols + # other than HTTP, HTTPS or FTP. + newurl_lower = newurl.lower() + if not (newurl_lower.startswith('http://') or + newurl_lower.startswith('https://') or + newurl_lower.startswith('ftp://')): + raise IOError('redirect error', errcode, + errmsg + " - Redirection to url '%s' is not allowed" % + newurl, + headers) + return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Lib/urllib2.py b/Lib/urllib2.py --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -578,6 +578,17 @@ newurl = urlparse.urljoin(req.get_full_url(), newurl) + # For security reasons we do not allow redirects to protocols + # other than HTTP, HTTPS or FTP. + newurl_lower = newurl.lower() + if not (newurl_lower.startswith('http://') or + newurl_lower.startswith('https://') or + newurl_lower.startswith('ftp://')): + raise HTTPError(newurl, code, + msg + " - Redirection to url '%s' is not allowed" % + newurl, + headers, fp) + # XXX Probably want to forget about the state of the current # request, although that might interact poorly with other # handlers that also use handler-specific request attributes diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,8 @@ *NOTE: Python 2.6 is in security-fix-only mode. No non-security bug fixes are allowed. Python 2.6.7 and beyond will be source only releases.* +- Issue #11662: Make urllib and urllib2 ignore redirections if the + scheme is not HTTP, HTTPS or FTP (CVE-2011-1521). Core and Builtins ----------------- -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.5): Adding .hgignore (copied from default branch).
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/92293101839c changeset: 69044:92293101839c branch: 2.5 user: guido(a)google.com date: Tue Mar 29 09:53:33 2011 -0700 summary: Adding .hgignore (copied from default branch). files: .hgignore | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 66 insertions(+), 0 deletions(-) diff --git a/.hgignore b/.hgignore new file mode 100644 --- /dev/null +++ b/.hgignore @@ -0,0 +1,66 @@ +.gdb_history +.purify +.svn/ +Makefile$ +Makefile.pre$ +TAGS$ +autom4te.cache$ +build/ +buildno$ +config.cache +config.log +config.status +config.status.lineno +db_home +platform$ +pyconfig.h$ +python$ +python.exe$ +reflog.txt$ +tags$ +Lib/plat-mac/errors.rsrc.df.rsrc +Doc/tools/sphinx/ +Doc/tools/docutils/ +Doc/tools/jinja/ +Doc/tools/jinja2/ +Doc/tools/pygments/ +Misc/python.pc +Modules/Setup$ +Modules/Setup.config +Modules/Setup.local +Modules/config.c +Modules/ld_so_aix$ +Parser/pgen$ +Parser/pgen.stamp$ +^core +^python-gdb.py +^python.exe-gdb.py +^pybuilddir.txt + +syntax: glob +libpython*.a +*.swp +*.o +*.pyc +*.pyo +*.pyd +*.cover +*.orig +*.rej +*~ +Lib/lib2to3/*.pickle +Lib/test/data/* +Misc/*.wpu +PC/python_nt*.h +PC/pythonnt_rc*.h +PC/*.obj +PCbuild/*.exe +PCbuild/*.dll +PCbuild/*.pdb +PCbuild/*.lib +PCbuild/*.exp +PCbuild/*.o +PCbuild/*.ncb +PCbuild/*.bsc +PCbuild/Win32-temp-* +__pycache__ -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.5): Add CVE number to urllib/urllib2 news item.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/f03e2acb9826 changeset: 69043:f03e2acb9826 branch: 2.5 user: guido(a)google.com date: Mon Mar 28 13:53:40 2011 -0700 summary: Add CVE number to urllib/urllib2 news item. files: Misc/NEWS | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -13,7 +13,7 @@ ------- - Issue #11662: Make urllib and urllib2 ignore redirections if the - scheme is not HTTP, HTTPS or FTP. This fixes a security hole. + scheme is not HTTP, HTTPS or FTP (CVE-2011-1521). - Issue #8674: Fixed a number of incorrect or undefined-behaviour-inducing overflow checks in the audioop module (CVE-2010-1634). -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.5): Add FTP to the allowed url schemes. Add Misc/NEWS.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/ca3b117c40f3 changeset: 69041:ca3b117c40f3 branch: 2.5 user: guido(a)google.com date: Thu Mar 24 10:44:17 2011 -0700 summary: Add FTP to the allowed url schemes. Add Misc/NEWS. files: Lib/urllib.py | 5 +++-- Lib/urllib2.py | 5 +++-- Misc/NEWS | 3 +++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/urllib.py b/Lib/urllib.py --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -643,10 +643,11 @@ newurl = basejoin(self.type + ":" + url, newurl) # For security reasons we do not allow redirects to protocols - # other than HTTP or HTTPS. + # other than HTTP, HTTPS or FTP. newurl_lower = newurl.lower() if not (newurl_lower.startswith('http://') or - newurl_lower.startswith('https://')): + newurl_lower.startswith('https://') or + newurl_lower.startswith('ftp://')): return void = fp.read() diff --git a/Lib/urllib2.py b/Lib/urllib2.py --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -556,10 +556,11 @@ newurl = urlparse.urljoin(req.get_full_url(), newurl) # For security reasons we do not allow redirects to protocols - # other than HTTP or HTTPS. + # other than HTTP, HTTPS or FTP. newurl_lower = newurl.lower() if not (newurl_lower.startswith('http://') or - newurl_lower.startswith('https://')): + newurl_lower.startswith('https://') or + newurl_lower.startswith('ftp://')): return # XXX Probably want to forget about the state of the current diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,9 @@ Library ------- +- Issue #11662: Make urllib and urllib2 ignore redirections if the + scheme is not HTTP, HTTPS or FTP. This fixes a security hole. + - Issue #8674: Fixed a number of incorrect or undefined-behaviour-inducing overflow checks in the audioop module (CVE-2010-1634). -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.5): Add tests for the urllib[2] vulnerability. Change to raise exceptions.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/9d06d5eb1a7e changeset: 69042:9d06d5eb1a7e branch: 2.5 user: guido(a)google.com date: Mon Mar 28 13:47:01 2011 -0700 summary: Add tests for the urllib[2] vulnerability. Change to raise exceptions. files: Lib/test/test_urllib.py | 14 ++++++++++++++ Lib/test/test_urllib2.py | 21 +++++++++++++++++++++ Lib/urllib.py | 10 ++++++---- Lib/urllib2.py | 5 ++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -122,6 +122,20 @@ finally: self.unfakehttp() + def test_invalid_redirect(self): + # urlopen() should raise IOError for many error codes. + self.fakehttp("""HTTP/1.1 302 Found +Date: Wed, 02 Jan 2008 03:03:54 GMT +Server: Apache/1.3.33 (Debian GNU/Linux) mod_ssl/2.8.22 OpenSSL/0.9.7e +Location: file:README +Connection: close +Content-Type: text/html; charset=iso-8859-1 +""") + try: + self.assertRaises(IOError, urllib.urlopen, "http://python.org/") + finally: + self.unfakehttp() + def test_empty_socket(self): """urlopen() raises IOError if the underlying socket does not send any data. (#1680230) """ diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -857,6 +857,27 @@ self.assertEqual(count, urllib2.HTTPRedirectHandler.max_redirections) + def test_invalid_redirect(self): + from_url = "http://example.com/a.html" + valid_schemes = ['http', 'https', 'ftp'] + invalid_schemes = ['file', 'imap', 'ldap'] + schemeless_url = "example.com/b.html" + h = urllib2.HTTPRedirectHandler() + o = h.parent = MockOpener() + req = Request(from_url) + + for scheme in invalid_schemes: + invalid_url = scheme + '://' + schemeless_url + self.assertRaises(urllib2.HTTPError, h.http_error_302, + req, MockFile(), 302, "Security Loophole", + MockHeaders({"location": invalid_url})) + + for scheme in valid_schemes: + valid_url = scheme + '://' + schemeless_url + h.http_error_302(req, MockFile(), 302, "That's fine", + MockHeaders({"location": valid_url})) + self.assertEqual(o.req.get_full_url(), valid_url) + def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from cookielib import CookieJar diff --git a/Lib/urllib.py b/Lib/urllib.py --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -638,7 +638,8 @@ newurl = headers['uri'] else: return - + void = fp.read() + fp.close() # In case the server sent a relative URL, join with original: newurl = basejoin(self.type + ":" + url, newurl) @@ -648,10 +649,11 @@ if not (newurl_lower.startswith('http://') or newurl_lower.startswith('https://') or newurl_lower.startswith('ftp://')): - return + raise IOError('redirect error', errcode, + errmsg + " - Redirection to url '%s' is not allowed" % + newurl, + headers) - void = fp.read() - fp.close() return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Lib/urllib2.py b/Lib/urllib2.py --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -561,7 +561,10 @@ if not (newurl_lower.startswith('http://') or newurl_lower.startswith('https://') or newurl_lower.startswith('ftp://')): - return + raise HTTPError(newurl, code, + msg + " - Redirection to url '%s' is not allowed" % + newurl, + headers, fp) # XXX Probably want to forget about the state of the current # request, although that might interact poorly with other -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.5): Issue 22663: fix redirect vulnerability in urllib/urllib2.
by guido.van.rossum March 29, 2011

March 29, 2011

http://hg.python.org/cpython/rev/dd852a0f92d6 changeset: 69040:dd852a0f92d6 branch: 2.5 parent: 68801:f9763c363cc3 user: guido(a)google.com date: Thu Mar 24 08:07:45 2011 -0700 summary: Issue 22663: fix redirect vulnerability in urllib/urllib2. files: Lib/urllib.py | 13 +++++++++++-- Lib/urllib2.py | 7 +++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/urllib.py b/Lib/urllib.py --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -638,10 +638,19 @@ newurl = headers['uri'] else: return + + # In case the server sent a relative URL, join with original: + newurl = basejoin(self.type + ":" + url, newurl) + + # For security reasons we do not allow redirects to protocols + # other than HTTP or HTTPS. + newurl_lower = newurl.lower() + if not (newurl_lower.startswith('http://') or + newurl_lower.startswith('https://')): + return + void = fp.read() fp.close() - # In case the server sent a relative URL, join with original: - newurl = basejoin(self.type + ":" + url, newurl) return self.open(newurl) def http_error_301(self, url, fp, errcode, errmsg, headers, data=None): diff --git a/Lib/urllib2.py b/Lib/urllib2.py --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -555,6 +555,13 @@ return newurl = urlparse.urljoin(req.get_full_url(), newurl) + # For security reasons we do not allow redirects to protocols + # other than HTTP or HTTPS. + newurl_lower = newurl.lower() + if not (newurl_lower.startswith('http://') or + newurl_lower.startswith('https://')): + return + # XXX Probably want to forget about the state of the current # request, although that might interact poorly with other # handlers that also use handler-specific request attributes -- Repository URL: http://hg.python.org/cpython

1 0