https://github.com/python/cpython/commit/bd48280cb66544827952ca91e326cbb178c... commit: bd48280cb66544827952ca91e326cbb178c8c461 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> committer: Serhiy Storchaka <storchaka@gmail.com> date: 2019-03-30T15:52:41+02:00 summary: bpo-24214: Fixed the UTF-8 incremental decoder. (GH-12603) (GH-12627) The bug occurred when the encoded surrogate character is passed to the incremental decoder in two chunks. (cherry picked from commit 7a465cb5ee7e298cae626ace1fc3e7d97df79f2e) Co-authored-by: Serhiy Storchaka <storchaka@gmail.com> files: A Misc/NEWS.d/next/Core and Builtins/2019-03-28-15-22-45.bpo-24214.tZ6lYU.rst M Lib/test/test_codecs.py M Objects/unicodeobject.c diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 293dfbc61aba..5ba2c7bdc5f8 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -401,6 +401,15 @@ def test_lone_surrogates(self): self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"), before + backslashreplace + after) + def test_incremental_surrogatepass(self): + # Test incremental decoder for surrogatepass handler: + # see issue #24214 + data = '\uD901'.encode(self.encoding, 'surrogatepass') + for i in range(1, len(data)): + dec = codecs.getincrementaldecoder(self.encoding)('surrogatepass') + self.assertEqual(dec.decode(data[:i]), '') + self.assertEqual(dec.decode(data[i:], True), '\uD901') + class UTF32Test(ReadTest, unittest.TestCase): encoding = "utf-32" diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-03-28-15-22-45.bpo-24214.tZ6lYU.rst b/Misc/NEWS.d/next/Core and Builtins/2019-03-28-15-22-45.bpo-24214.tZ6lYU.rst new file mode 100644 index 000000000000..abb27591f78a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-03-28-15-22-45.bpo-24214.tZ6lYU.rst @@ -0,0 +1,2 @@ +Fixed support of the surrogatepass error handler in the UTF-8 incremental +decoder. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index adcf69d4e539..e18937981bb7 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4890,6 +4890,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, case 2: case 3: case 4: + if (s == end || consumed) { + goto End; + } errmsg = "invalid continuation byte"; startinpos = s - starts; endinpos = startinpos + ch - 1;