[Python-checkins] bpo-33618: Enable TLS 1.3 in tests (GH-7079)
Christian Heimes
webhook-mailer at python.org
Wed May 23 16:24:54 EDT 2018
https://github.com/python/cpython/commit/529525fb5a8fd9b96ab4021311a598c77588b918
commit: 529525fb5a8fd9b96ab4021311a598c77588b918
branch: master
author: Christian Heimes <christian at python.org>
committer: GitHub <noreply at github.com>
date: 2018-05-23T22:24:45+02:00
summary:
bpo-33618: Enable TLS 1.3 in tests (GH-7079)
TLS 1.3 behaves slightly different than TLS 1.2. Session tickets and TLS
client cert auth are now handled after the initialy handshake. Tests now
either send/recv data to trigger session and client certs. Or tests
ignore ConnectionResetError / BrokenPipeError on the server side to
handle clients that force-close the socket fd.
To test TLS 1.3, OpenSSL 1.1.1-pre7-dev (git master + OpenSSL PR
https://github.com/openssl/openssl/pull/6340) is required.
Signed-off-by: Christian Heimes <christian at python.org>
files:
A Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst
M Doc/library/ssl.rst
M Doc/whatsnew/3.7.rst
M Lib/test/test_asyncio/test_sslproto.py
M Lib/test/test_asyncio/utils.py
M Lib/test/test_ftplib.py
M Lib/test/test_poplib.py
M Lib/test/test_ssl.py
M Tools/ssl/multissltests.py
diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst
index 2ccea13b6142..14eac2c58947 100644
--- a/Doc/library/ssl.rst
+++ b/Doc/library/ssl.rst
@@ -2587,7 +2587,33 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or
:func:`~ssl.RAND_pseudo_bytes` is sufficient.
-.. ssl-libressl:
+.. _ssl-tlsv1_3:
+
+TLS 1.3
+-------
+
+.. versionadded:: 3.7
+
+Python has provisional and experimental support for TLS 1.3 with OpenSSL
+1.1.1. The new protocol behaves slightly differently than previous version
+of TLS/SSL. Some new TLS 1.3 features are not yet available.
+
+- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and
+ ChaCha20 cipher suites are enabled by default. The method
+ :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3
+ ciphers yet, but :meth:`SSLContext.get_cipers` returns them.
+- Session tickets are no longer sent as part of the initial handshake and
+ are handled differently. :attr:`SSLSocket.session` and :class:`SSLSession`
+ are not compatible with TLS 1.3.
+- Client-side certificates are also no longer verified during the initial
+ handshake. A server can request a certificate at any time. Clients
+ process certificate requests while they send or receive application data
+ from the server.
+- TLS 1.3 features like early data, deferred TLS client cert request,
+ signature algorithm configuration, and rekeying are not supported yet.
+
+
+.. _ssl-libressl:
LibreSSL support
----------------
diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst
index af2aad9e81e7..46015af3e739 100644
--- a/Doc/whatsnew/3.7.rst
+++ b/Doc/whatsnew/3.7.rst
@@ -1244,8 +1244,8 @@ Host name validation can be customized with
.. note::
The improved host name check requires a *libssl* implementation compatible
with OpenSSL 1.0.2 or 1.1. Consequently, OpenSSL 0.9.8 and 1.0.1 are no
- longer supported and LibreSSL is temporarily not supported until it gains
- the necessary OpenSSL 1.0.2 APIs.
+ longer supported. The ssl module is mostly compatible with LibreSSL 2.7.2
+ and newer.
The ``ssl`` module no longer sends IP addresses in SNI TLS extension.
(Contributed by Christian Heimes in :issue:`32185`.)
@@ -1270,8 +1270,12 @@ rather than the U-label form (``"pythön.org"``). (Contributed by
Nathaniel J. Smith and Christian Heimes in :issue:`28414`.)
The ``ssl`` module has preliminary and experimental support for TLS 1.3 and
-OpenSSL 1.1.1. (Contributed by Christian Heimes in :issue:`32947`,
-:issue:`20995`, :issue:`29136`, and :issue:`30622`)
+OpenSSL 1.1.1. At the time of Python 3.7.0 release, OpenSSL 1.1.1 is still
+under development and TLS 1.3 hasn't been finalized yet. The TLS 1.3
+handshake and protocol behaves slightly differently than TLS 1.2 and earlier,
+see :ref:`ssl-tlsv1_3`.
+(Contributed by Christian Heimes in :issue:`32947`, :issue:`20995`,
+:issue:`29136`, :issue:`30622` and :issue:`33618`)
:class:`~ssl.SSLSocket` and :class:`~ssl.SSLObject` no longer have a public
constructor. Direct instantiation was never a documented and supported
diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py
index 7b27f4cfa322..c534a341352b 100644
--- a/Lib/test/test_asyncio/test_sslproto.py
+++ b/Lib/test/test_asyncio/test_sslproto.py
@@ -251,6 +251,8 @@ def test_start_tls_server_1(self):
server_context = test_utils.simple_server_sslcontext()
client_context = test_utils.simple_client_sslcontext()
+ # TODO: fix TLSv1.3 support
+ client_context.options |= ssl.OP_NO_TLSv1_3
def client(sock, addr):
time.sleep(0.5)
diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py
index 711085fde5c5..96dfe2f85b4d 100644
--- a/Lib/test/test_asyncio/utils.py
+++ b/Lib/test/test_asyncio/utils.py
@@ -74,8 +74,6 @@ def simple_server_sslcontext():
server_context.load_cert_chain(ONLYCERT, ONLYKEY)
server_context.check_hostname = False
server_context.verify_mode = ssl.CERT_NONE
- # TODO: fix TLSv1.3 support
- server_context.options |= ssl.OP_NO_TLSv1_3
return server_context
diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py
index 1a8e2f91d386..f9488a96cfca 100644
--- a/Lib/test/test_ftplib.py
+++ b/Lib/test/test_ftplib.py
@@ -257,6 +257,7 @@ class DummyFTPServer(asyncore.dispatcher, threading.Thread):
def __init__(self, address, af=socket.AF_INET):
threading.Thread.__init__(self)
asyncore.dispatcher.__init__(self)
+ self.daemon = True
self.create_socket(af, socket.SOCK_STREAM)
self.bind(address)
self.listen(5)
@@ -312,8 +313,6 @@ class SSLConnection(asyncore.dispatcher):
def secure_connection(self):
context = ssl.SSLContext()
- # TODO: fix TLSv1.3 support
- context.options |= ssl.OP_NO_TLSv1_3
context.load_cert_chain(CERTFILE)
socket = context.wrap_socket(self.socket,
suppress_ragged_eofs=False,
@@ -405,7 +404,7 @@ def handle_error(self):
def close(self):
if (isinstance(self.socket, ssl.SSLSocket) and
- self.socket._sslobj is not None):
+ self.socket._sslobj is not None):
self._do_ssl_shutdown()
else:
super(SSLConnection, self).close()
@@ -910,8 +909,6 @@ def test_auth_issued_twice(self):
def test_context(self):
self.client.quit()
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
- # TODO: fix TLSv1.3 support
- ctx.options |= ssl.OP_NO_TLSv1_3
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
self.assertRaises(ValueError, ftplib.FTP_TLS, keyfile=CERTFILE,
@@ -944,8 +941,6 @@ def test_ccc(self):
def test_check_hostname(self):
self.client.quit()
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
- # TODO: fix TLSv1.3 support
- ctx.options |= ssl.OP_NO_TLSv1_3
self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
self.assertEqual(ctx.check_hostname, True)
ctx.load_verify_locations(CAFILE)
@@ -982,6 +977,7 @@ def setUp(self):
self.sock.settimeout(20)
self.port = support.bind_port(self.sock)
self.server_thread = threading.Thread(target=self.server)
+ self.server_thread.daemon = True
self.server_thread.start()
# Wait for the server to be ready.
self.evt.wait()
diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py
index bbedbbdb10a1..20d4eeac12d4 100644
--- a/Lib/test/test_poplib.py
+++ b/Lib/test/test_poplib.py
@@ -153,8 +153,6 @@ def cmd_stls(self, arg):
if self.tls_active is False:
self.push('+OK Begin TLS negotiation')
context = ssl.SSLContext()
- # TODO: fix TLSv1.3 support
- context.options |= ssl.OP_NO_TLSv1_3
context.load_cert_chain(CERTFILE)
tls_sock = context.wrap_socket(self.socket,
server_side=True,
@@ -206,6 +204,7 @@ class DummyPOP3Server(asyncore.dispatcher, threading.Thread):
def __init__(self, address, af=socket.AF_INET):
threading.Thread.__init__(self)
asyncore.dispatcher.__init__(self)
+ self.daemon = True
self.create_socket(af, socket.SOCK_STREAM)
self.bind(address)
self.listen(5)
@@ -370,8 +369,6 @@ def test_stls(self):
def test_stls_context(self):
expected = b'+OK Begin TLS negotiation'
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
- # TODO: fix TLSv1.3 support
- ctx.options |= ssl.OP_NO_TLSv1_3
ctx.load_verify_locations(CAFILE)
self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED)
self.assertEqual(ctx.check_hostname, True)
@@ -412,8 +409,6 @@ def test__all__(self):
def test_context(self):
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
- # TODO: fix TLSv1.3 support
- ctx.options |= ssl.OP_NO_TLSv1_3
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
self.assertRaises(ValueError, poplib.POP3_SSL, self.server.host,
@@ -482,7 +477,7 @@ def setUp(self):
self.sock.settimeout(60) # Safety net. Look issue 11812
self.port = test_support.bind_port(self.sock)
self.thread = threading.Thread(target=self.server, args=(self.evt,self.sock))
- self.thread.setDaemon(True)
+ self.thread.daemon = True
self.thread.start()
self.evt.wait()
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 03952b10a8e7..73d3e3bbcdae 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -1826,6 +1826,7 @@ def test_connect_capath(self):
s.connect(self.server_addr)
cert = s.getpeercert()
self.assertTrue(cert)
+
# Same with a bytes `capath` argument
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
ctx.verify_mode = ssl.CERT_REQUIRED
@@ -1841,8 +1842,6 @@ def test_connect_cadata(self):
der = ssl.PEM_cert_to_DER_cert(pem)
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
ctx.verify_mode = ssl.CERT_REQUIRED
- # TODO: fix TLSv1.3 support
- ctx.options |= ssl.OP_NO_TLSv1_3
ctx.load_verify_locations(cadata=pem)
with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
s.connect(self.server_addr)
@@ -1852,8 +1851,6 @@ def test_connect_cadata(self):
# same with DER
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
ctx.verify_mode = ssl.CERT_REQUIRED
- # TODO: fix TLSv1.3 support
- ctx.options |= ssl.OP_NO_TLSv1_3
ctx.load_verify_locations(cadata=der)
with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s:
s.connect(self.server_addr)
@@ -2109,11 +2106,21 @@ def wrap_conn(self):
self.sock, server_side=True)
self.server.selected_npn_protocols.append(self.sslconn.selected_npn_protocol())
self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol())
- except (ssl.SSLError, ConnectionResetError, OSError) as e:
+ except (ConnectionResetError, BrokenPipeError) as e:
# We treat ConnectionResetError as though it were an
# SSLError - OpenSSL on Ubuntu abruptly closes the
# connection when asked to use an unsupported protocol.
#
+ # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL
+ # tries to send session tickets after handshake.
+ # https://github.com/openssl/openssl/issues/6342
+ self.server.conn_errors.append(str(e))
+ if self.server.chatty:
+ handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n")
+ self.running = False
+ self.close()
+ return False
+ except (ssl.SSLError, OSError) as e:
# OSError may occur with wrong protocols, e.g. both
# sides use PROTOCOL_TLS_SERVER.
#
@@ -2220,11 +2227,22 @@ def run(self):
sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n"
% (msg, ctype, msg.lower(), ctype))
self.write(msg.lower())
+ except ConnectionResetError:
+ # XXX: OpenSSL 1.1.1 sometimes raises ConnectionResetError
+ # when connection is not shut down gracefully.
+ if self.server.chatty and support.verbose:
+ sys.stdout.write(
+ " Connection reset by peer: {}\n".format(
+ self.addr)
+ )
+ self.close()
+ self.running = False
except OSError:
if self.server.chatty:
handle_error("Test server failure:\n")
self.close()
self.running = False
+
# normally, we'd just stop here, but for the test
# harness, we want to stop the server
self.server.stop()
@@ -2299,6 +2317,11 @@ def run(self):
pass
except KeyboardInterrupt:
self.stop()
+ except BaseException as e:
+ if support.verbose and self.chatty:
+ sys.stdout.write(
+ ' connection handling failed: ' + repr(e) + '\n')
+
self.sock.close()
def stop(self):
@@ -2745,8 +2768,6 @@ def test_check_hostname_idn(self):
server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
server_context.load_cert_chain(IDNSANSFILE)
- # TODO: fix TLSv1.3 support
- server_context.options |= ssl.OP_NO_TLSv1_3
context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
context.verify_mode = ssl.CERT_REQUIRED
@@ -2797,7 +2818,7 @@ def test_check_hostname_idn(self):
with self.assertRaises(ssl.CertificateError):
s.connect((HOST, server.port))
- def test_wrong_cert(self):
+ def test_wrong_cert_tls12(self):
"""Connecting when the server rejects the client's certificate
Launch a server with CERT_REQUIRED, and check that trying to
@@ -2808,9 +2829,8 @@ def test_wrong_cert(self):
client_context.load_cert_chain(WRONG_CERT)
# require TLS client authentication
server_context.verify_mode = ssl.CERT_REQUIRED
- # TODO: fix TLSv1.3 support
- # With TLS 1.3, test fails with exception in server thread
- server_context.options |= ssl.OP_NO_TLSv1_3
+ # TLS 1.3 has different handshake
+ client_context.maximum_version = ssl.TLSVersion.TLSv1_2
server = ThreadedEchoServer(
context=server_context, chatty=True, connectionchatty=True,
@@ -2835,6 +2855,36 @@ def test_wrong_cert(self):
else:
self.fail("Use of invalid cert should have failed!")
+ @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3")
+ def test_wrong_cert_tls13(self):
+ client_context, server_context, hostname = testing_context()
+ client_context.load_cert_chain(WRONG_CERT)
+ server_context.verify_mode = ssl.CERT_REQUIRED
+ server_context.minimum_version = ssl.TLSVersion.TLSv1_3
+ client_context.minimum_version = ssl.TLSVersion.TLSv1_3
+
+ server = ThreadedEchoServer(
+ context=server_context, chatty=True, connectionchatty=True,
+ )
+ with server, \
+ client_context.wrap_socket(socket.socket(),
+ server_hostname=hostname) as s:
+ # TLS 1.3 perform client cert exchange after handshake
+ s.connect((HOST, server.port))
+ try:
+ s.write(b'data')
+ s.read(4)
+ except ssl.SSLError as e:
+ if support.verbose:
+ sys.stdout.write("\nSSLError is %r\n" % e)
+ except OSError as e:
+ if e.errno != errno.ECONNRESET:
+ raise
+ if support.verbose:
+ sys.stdout.write("\nsocket.error is %r\n" % e)
+ else:
+ self.fail("Use of invalid cert should have failed!")
+
def test_rude_shutdown(self):
"""A brutal shutdown of an SSL server should raise an OSError
in the client when attempting handshake.
@@ -3405,7 +3455,7 @@ def serve():
# Block on the accept and wait on the connection to close.
evt.set()
remote, peer = server.accept()
- remote.recv(1)
+ remote.send(remote.recv(4))
t = threading.Thread(target=serve)
t.start()
@@ -3413,6 +3463,8 @@ def serve():
evt.wait()
client = context.wrap_socket(socket.socket())
client.connect((host, port))
+ client.send(b'data')
+ client.recv()
client_addr = client.getsockname()
client.close()
t.join()
@@ -3465,7 +3517,7 @@ def test_version_basic(self):
self.assertIs(s.version(), None)
self.assertIs(s._sslobj, None)
s.connect((HOST, server.port))
- if ssl.OPENSSL_VERSION_INFO >= (1, 1, 1):
+ if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3:
self.assertEqual(s.version(), 'TLSv1.3')
elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2):
self.assertEqual(s.version(), 'TLSv1.2')
@@ -3574,8 +3626,6 @@ def test_tls_unique_channel_binding(self):
sys.stdout.write("\n")
client_context, server_context, hostname = testing_context()
- # TODO: fix TLSv1.3 support
- client_context.options |= ssl.OP_NO_TLSv1_3
server = ThreadedEchoServer(context=server_context,
chatty=True,
@@ -3594,7 +3644,10 @@ def test_tls_unique_channel_binding(self):
# check if it is sane
self.assertIsNotNone(cb_data)
- self.assertEqual(len(cb_data), 12) # True for TLSv1
+ if s.version() == 'TLSv1.3':
+ self.assertEqual(len(cb_data), 48)
+ else:
+ self.assertEqual(len(cb_data), 12) # True for TLSv1
# and compare with the peers version
s.write(b"CB tls-unique\n")
@@ -3616,7 +3669,10 @@ def test_tls_unique_channel_binding(self):
# is it really unique
self.assertNotEqual(cb_data, new_cb_data)
self.assertIsNotNone(cb_data)
- self.assertEqual(len(cb_data), 12) # True for TLSv1
+ if s.version() == 'TLSv1.3':
+ self.assertEqual(len(cb_data), 48)
+ else:
+ self.assertEqual(len(cb_data), 12) # True for TLSv1
s.write(b"CB tls-unique\n")
peer_data_repr = s.read().strip()
self.assertEqual(peer_data_repr,
diff --git a/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst b/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst
new file mode 100644
index 000000000000..6cc2452b145c
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-05-23-20-14-34.bpo-33618.xU39lr.rst
@@ -0,0 +1,2 @@
+Finalize and document preliminary and experimental TLS 1.3 support with
+OpenSSL 1.1.1
diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py
index bbc5c6652098..c4ebe317797d 100755
--- a/Tools/ssl/multissltests.py
+++ b/Tools/ssl/multissltests.py
@@ -47,7 +47,7 @@
OPENSSL_RECENT_VERSIONS = [
"1.0.2o",
"1.1.0h",
- "1.1.1-pre6",
+ # "1.1.1-pre7",
]
LIBRESSL_OLD_VERSIONS = [
@@ -73,7 +73,7 @@
parser.add_argument(
'--debug',
action='store_true',
- help="Enable debug mode",
+ help="Enable debug logging",
)
parser.add_argument(
'--disable-ancient',
@@ -130,6 +130,18 @@
default='',
help="Override the automatic system type detection."
)
+parser.add_argument(
+ '--force',
+ action='store_true',
+ dest='force',
+ help="Force build and installation."
+)
+parser.add_argument(
+ '--keep-sources',
+ action='store_true',
+ dest='keep_sources',
+ help="Keep original sources for debugging."
+)
class AbstractBuilder(object):
@@ -260,26 +272,31 @@ def _build_src(self):
"""Now build openssl"""
log.info("Running build in {}".format(self.build_dir))
cwd = self.build_dir
- cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)]
- env = None
+ cmd = [
+ "./config",
+ "shared", "--debug",
+ "--prefix={}".format(self.install_dir)
+ ]
+ env = os.environ.copy()
+ # set rpath
+ env["LD_RUN_PATH"] = self.lib_dir
if self.system:
- env = os.environ.copy()
env['SYSTEM'] = self.system
self._subprocess_call(cmd, cwd=cwd, env=env)
# Old OpenSSL versions do not support parallel builds.
self._subprocess_call(["make", "-j1"], cwd=cwd, env=env)
- def _make_install(self, remove=True):
+ def _make_install(self):
self._subprocess_call(
["make", "-j1", self.install_target],
cwd=self.build_dir
)
- if remove:
+ if not self.args.keep_sources:
shutil.rmtree(self.build_dir)
def install(self):
log.info(self.openssl_cli)
- if not self.has_openssl:
+ if not self.has_openssl or self.args.force:
if not self.has_src:
self._download_src()
else:
More information about the Python-checkins
mailing list