[Python-checkins] [3.7] bpo-36520: Email header folded incorrectly (GH-13608) (GH-13910)
Miss Islington (bot)
webhook-mailer at python.org
Tue Jun 11 19:27:17 EDT 2019
https://github.com/python/cpython/commit/0745cc66db3acbe7951073071cf063db6337dd10
commit: 0745cc66db3acbe7951073071cf063db6337dd10
branch: 3.7
author: Abhilash Raj <maxking at users.noreply.github.com>
committer: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
date: 2019-06-11T16:27:06-07:00
summary:
[3.7] bpo-36520: Email header folded incorrectly (GH-13608) (GH-13910)
* [bpo-36520](https://bugs.python.org/issue36520): reset the encoded word offset when starting a new
line during an email header folding operation
* 📜🤖 Added by blurb_it.
* [bpo-36520](https://bugs.python.org/issue36520): add an additional test case, and provide descriptive
comments for the test_folding_with_utf8_encoding_* tests
* [bpo-36520](https://bugs.python.org/issue36520): fix whitespace issue
* [bpo-36520](https://bugs.python.org/issue36520): changes per reviewer request -- remove extraneous
backslashes; add whitespace between terminating quotes and
line-continuation backslashes; use "bpo-" instead of
"issue GH-" in comments
(cherry picked from commit f6713e84afc5addcfa8477dbdf2c027787f711c0)
Co-authored-by: websurfer5 <49998481+websurfer5 at users.noreply.github.com>
https://bugs.python.org/issue36520
files:
A Misc/NEWS.d/next/Library/2019-05-28-02-37-00.bpo-36520.W4tday.rst
M Lib/email/_header_value_parser.py
M Lib/test/test_email/test_message.py
diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py
index 18aecbffa71a..fc00b4a098cb 100644
--- a/Lib/email/_header_value_parser.py
+++ b/Lib/email/_header_value_parser.py
@@ -2683,6 +2683,7 @@ def _refold_parse_tree(parse_tree, *, policy):
newline = _steal_trailing_WSP_if_exists(lines)
if newline or part.startswith_fws():
lines.append(newline + tstr)
+ last_ew = None
continue
if not hasattr(part, 'encode'):
# It's not a terminal, try folding the subparts.
diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py
index f3a57df9e9cf..5dc46e1b812c 100644
--- a/Lib/test/test_email/test_message.py
+++ b/Lib/test/test_email/test_message.py
@@ -784,6 +784,137 @@ def test_str_defaults_to_utf8(self):
m['Subject'] = 'unicöde'
self.assertEqual(str(m), 'Subject: unicöde\n\n')
+ def test_folding_with_utf8_encoding_1(self):
+ # bpo-36520
+ #
+ # Fold a line that contains UTF-8 words before
+ # and after the whitespace fold point, where the
+ # line length limit is reached within an ASCII
+ # word.
+
+ m = EmailMessage()
+ m['Subject'] = 'Hello Wörld! Hello Wörld! ' \
+ 'Hello Wörld! Hello Wörld!Hello Wörld!'
+ self.assertEqual(bytes(m),
+ b'Subject: Hello =?utf-8?q?W=C3=B6rld!_Hello_W'
+ b'=C3=B6rld!_Hello_W=C3=B6rld!?=\n'
+ b' Hello =?utf-8?q?W=C3=B6rld!Hello_W=C3=B6rld!?=\n\n')
+
+
+ def test_folding_with_utf8_encoding_2(self):
+ # bpo-36520
+ #
+ # Fold a line that contains UTF-8 words before
+ # and after the whitespace fold point, where the
+ # line length limit is reached at the end of an
+ # encoded word.
+
+ m = EmailMessage()
+ m['Subject'] = 'Hello Wörld! Hello Wörld! ' \
+ 'Hello Wörlds123! Hello Wörld!Hello Wörld!'
+ self.assertEqual(bytes(m),
+ b'Subject: Hello =?utf-8?q?W=C3=B6rld!_Hello_W'
+ b'=C3=B6rld!_Hello_W=C3=B6rlds123!?=\n'
+ b' Hello =?utf-8?q?W=C3=B6rld!Hello_W=C3=B6rld!?=\n\n')
+
+ def test_folding_with_utf8_encoding_3(self):
+ # bpo-36520
+ #
+ # Fold a line that contains UTF-8 words before
+ # and after the whitespace fold point, where the
+ # line length limit is reached at the end of the
+ # first word.
+
+ m = EmailMessage()
+ m['Subject'] = 'Hello-Wörld!-Hello-Wörld!-Hello-Wörlds123! ' \
+ 'Hello Wörld!Hello Wörld!'
+ self.assertEqual(bytes(m), \
+ b'Subject: =?utf-8?q?Hello-W=C3=B6rld!-Hello-W'
+ b'=C3=B6rld!-Hello-W=C3=B6rlds123!?=\n'
+ b' Hello =?utf-8?q?W=C3=B6rld!Hello_W=C3=B6rld!?=\n\n')
+
+ def test_folding_with_utf8_encoding_4(self):
+ # bpo-36520
+ #
+ # Fold a line that contains UTF-8 words before
+ # and after the fold point, where the first
+ # word is UTF-8 and the fold point is within
+ # the word.
+
+ m = EmailMessage()
+ m['Subject'] = 'Hello-Wörld!-Hello-Wörld!-Hello-Wörlds123!-Hello' \
+ ' Wörld!Hello Wörld!'
+ self.assertEqual(bytes(m),
+ b'Subject: =?utf-8?q?Hello-W=C3=B6rld!-Hello-W'
+ b'=C3=B6rld!-Hello-W=C3=B6rlds123!?=\n'
+ b' =?utf-8?q?-Hello_W=C3=B6rld!Hello_W=C3=B6rld!?=\n\n')
+
+ def test_folding_with_utf8_encoding_5(self):
+ # bpo-36520
+ #
+ # Fold a line that contains a UTF-8 word after
+ # the fold point.
+
+ m = EmailMessage()
+ m['Subject'] = '123456789 123456789 123456789 123456789 123456789' \
+ ' 123456789 123456789 Hello Wörld!'
+ self.assertEqual(bytes(m),
+ b'Subject: 123456789 123456789 123456789 123456789'
+ b' 123456789 123456789 123456789\n'
+ b' Hello =?utf-8?q?W=C3=B6rld!?=\n\n')
+
+ def test_folding_with_utf8_encoding_6(self):
+ # bpo-36520
+ #
+ # Fold a line that contains a UTF-8 word before
+ # the fold point and ASCII words after
+
+ m = EmailMessage()
+ m['Subject'] = '123456789 123456789 123456789 123456789 Hello Wörld!' \
+ ' 123456789 123456789 123456789 123456789 123456789' \
+ ' 123456789'
+ self.assertEqual(bytes(m),
+ b'Subject: 123456789 123456789 123456789 123456789'
+ b' Hello =?utf-8?q?W=C3=B6rld!?=\n 123456789 '
+ b'123456789 123456789 123456789 123456789 '
+ b'123456789\n\n')
+
+ def test_folding_with_utf8_encoding_7(self):
+ # bpo-36520
+ #
+ # Fold a line twice that contains UTF-8 words before
+ # and after the first fold point, and ASCII words
+ # after the second fold point.
+
+ m = EmailMessage()
+ m['Subject'] = '123456789 123456789 Hello Wörld! Hello Wörld! ' \
+ '123456789-123456789 123456789 Hello Wörld! 123456789' \
+ ' 123456789'
+ self.assertEqual(bytes(m),
+ b'Subject: 123456789 123456789 Hello =?utf-8?q?'
+ b'W=C3=B6rld!_Hello_W=C3=B6rld!?=\n'
+ b' 123456789-123456789 123456789 Hello '
+ b'=?utf-8?q?W=C3=B6rld!?= 123456789\n 123456789\n\n')
+
+ def test_folding_with_utf8_encoding_8(self):
+ # bpo-36520
+ #
+ # Fold a line twice that contains UTF-8 words before
+ # the first fold point, and ASCII words after the
+ # first fold point, and UTF-8 words after the second
+ # fold point.
+
+ m = EmailMessage()
+ m['Subject'] = '123456789 123456789 Hello Wörld! Hello Wörld! ' \
+ '123456789 123456789 123456789 123456789 123456789 ' \
+ '123456789-123456789 123456789 Hello Wörld! 123456789' \
+ ' 123456789'
+ self.assertEqual(bytes(m),
+ b'Subject: 123456789 123456789 Hello '
+ b'=?utf-8?q?W=C3=B6rld!_Hello_W=C3=B6rld!?=\n 123456789 '
+ b'123456789 123456789 123456789 123456789 '
+ b'123456789-123456789\n 123456789 Hello '
+ b'=?utf-8?q?W=C3=B6rld!?= 123456789 123456789\n\n')
class TestMIMEPart(TestEmailMessageBase, TestEmailBase):
# Doing the full test run here may seem a bit redundant, since the two
diff --git a/Misc/NEWS.d/next/Library/2019-05-28-02-37-00.bpo-36520.W4tday.rst b/Misc/NEWS.d/next/Library/2019-05-28-02-37-00.bpo-36520.W4tday.rst
new file mode 100644
index 000000000000..8171bfe9e2df
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2019-05-28-02-37-00.bpo-36520.W4tday.rst
@@ -0,0 +1 @@
+Lengthy email headers with UTF-8 characters are now properly encoded when they are folded. Patch by Jeffrey Kintscher.
More information about the Python-checkins
mailing list