[Python-checkins] cpython (merge 3.4 -> 3.5): (Merge 3.5) Issue #23840: tokenize.open() now closes the temporary binary file

[victor.stinner]

https://hg.python.org/cpython/rev/a640d268ba97
changeset:   96286:a640d268ba97
branch:      3.5
parent:      96282:26a9d1fd78fc
parent:      96285:623e07ea43df
user:        Victor Stinner <victor.stinner at gmail.com>
date:        Tue May 26 00:46:44 2015 +0200
summary:
  (Merge 3.5) Issue #23840: tokenize.open() now closes the temporary binary file
on error to fix a resource warning.

files:
  Lib/test/test_tokenize.py |  10 +++++++++-
  Lib/tokenize.py           |  14 +++++++++-----
  Misc/NEWS                 |   3 +++
  3 files changed, 21 insertions(+), 6 deletions(-)


diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -834,7 +834,7 @@
                      STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
                      open as tokenize_open, Untokenizer)
 from io import BytesIO
-from unittest import TestCase
+from unittest import TestCase, mock
 import os, sys, glob
 import token
 
@@ -1246,6 +1246,14 @@
             ins = Bunk(lines, path)
             detect_encoding(ins.readline)
 
+    def test_open_error(self):
+        # Issue #23840: open() must close the binary file on error
+        m = BytesIO(b'#coding:xxx')
+        with mock.patch('tokenize._builtin_open', return_value=m):
+            self.assertRaises(SyntaxError, tokenize_open, 'foobar')
+        self.assertTrue(m.closed)
+
+
 
 class TestTokenize(TestCase):
 
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -435,11 +435,15 @@
     detect_encoding().
     """
     buffer = _builtin_open(filename, 'rb')
-    encoding, lines = detect_encoding(buffer.readline)
-    buffer.seek(0)
-    text = TextIOWrapper(buffer, encoding, line_buffering=True)
-    text.mode = 'r'
-    return text
+    try:
+        encoding, lines = detect_encoding(buffer.readline)
+        buffer.seek(0)
+        text = TextIOWrapper(buffer, encoding, line_buffering=True)
+        text.mode = 'r'
+        return text
+    except:
+        buffer.close()
+        raise
 
 
 def tokenize(readline):
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -323,6 +323,9 @@
 Library
 -------
 
+- Issue #23840: tokenize.open() now closes the temporary binary file on error
+  to fix a resource warning.
+
 - Issue #16914: new debuglevel 2 in smtplib adds timestamps to debug output.
 
 - Issue #7159: urllib.request now supports sending auth credentials

-- 
Repository URL: https://hg.python.org/cpython