Mailman 3 June 2014 - Python-checkins

cpython: Issue #21679: Prevent extraneous fstat() calls during open(). Patch by
by antoine.pitrou 30 Jun '14

30 Jun '14

http://hg.python.org/cpython/rev/3b5279b5bfd1 changeset: 91470:3b5279b5bfd1 user: Antoine Pitrou <solipsis(a)pitrou.net> date: Sun Jun 29 20:07:28 2014 -0400 summary: Issue #21679: Prevent extraneous fstat() calls during open(). Patch by Bohuslav Kabrda. files: Lib/test/test_fileio.py | 11 ++++++- Misc/NEWS | 3 + Modules/_io/_iomodule.c | 28 +++++----------- Modules/_io/fileio.c | 46 +++++++++++++++------------- 4 files changed, 47 insertions(+), 41 deletions(-) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -60,6 +60,15 @@ self.assertRaises((AttributeError, TypeError), setattr, f, attr, 'oops') + def testBlksize(self): + # test private _blksize attribute + blksize = io.DEFAULT_BUFFER_SIZE + # try to get preferred blksize from stat.st_blksize, if available + if hasattr(os, 'fstat'): + fst = os.fstat(self.f.fileno()) + blksize = getattr(fst, 'st_blksize', blksize) + self.assertEqual(self.f._blksize, blksize) + def testReadinto(self): # verify readinto self.f.write(bytes([1, 2])) @@ -141,7 +150,7 @@ def testOpendir(self): # Issue 3703: opening a directory should fill the errno # Windows always returns "[Errno 13]: Permission denied - # Unix calls dircheck() and returns "[Errno 21]: Is a directory" + # Unix uses fstat and returns "[Errno 21]: Is a directory" try: _FileIO('.', 'r') except OSError as e: diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -103,6 +103,9 @@ Library ------- +- Issue #21679: Prevent extraneous fstat() calls during open(). Patch by + Bohuslav Kabrda. + - Issue #21863: cProfile now displays the module name of C extension functions, in addition to their own name. diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -237,8 +237,8 @@ PyObject *raw, *modeobj = NULL, *buffer, *wrapper, *result = NULL; + _Py_IDENTIFIER(_blksize); _Py_IDENTIFIER(isatty); - _Py_IDENTIFIER(fileno); _Py_IDENTIFIER(mode); _Py_IDENTIFIER(close); @@ -380,24 +380,14 @@ line_buffering = 0; if (buffering < 0) { - buffering = DEFAULT_BUFFER_SIZE; -#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE - { - struct stat st; - long fileno; - PyObject *res = _PyObject_CallMethodId(raw, &PyId_fileno, NULL); - if (res == NULL) - goto error; - - fileno = PyLong_AsLong(res); - Py_DECREF(res); - if (fileno == -1 && PyErr_Occurred()) - goto error; - - if (fstat(fileno, &st) >= 0 && st.st_blksize > 1) - buffering = st.st_blksize; - } -#endif + PyObject *blksize_obj; + blksize_obj = _PyObject_GetAttrId(raw, &PyId__blksize); + if (blksize_obj == NULL) + goto error; + buffering = PyLong_AsLong(blksize_obj); + Py_DECREF(blksize_obj); + if (buffering == -1 && PyErr_Occurred()) + goto error; } if (buffering < 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -53,6 +53,7 @@ signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; char finalizing; + unsigned int blksize; PyObject *weakreflist; PyObject *dict; } fileio; @@ -161,6 +162,7 @@ self->writable = 0; self->appending = 0; self->seekable = -1; + self->blksize = 0; self->closefd = 1; self->weakreflist = NULL; } @@ -168,26 +170,6 @@ return (PyObject *) self; } -/* On Unix, open will succeed for directories. - In Python, there should be no file objects referring to - directories, so we need a check. */ - -static int -dircheck(fileio* self, PyObject *nameobj) -{ -#if defined(HAVE_FSTAT) && defined(S_ISDIR) && defined(EISDIR) - struct stat buf; - if (self->fd < 0) - return 0; - if (fstat(self->fd, &buf) == 0 && S_ISDIR(buf.st_mode)) { - errno = EISDIR; - PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, nameobj); - return -1; - } -#endif - return 0; -} - static int check_fd(int fd) { @@ -233,6 +215,9 @@ #elif !defined(MS_WINDOWS) int *atomic_flag_works = NULL; #endif +#ifdef HAVE_FSTAT + struct stat fdfstat; +#endif assert(PyFileIO_Check(oself)); if (self->fd >= 0) { @@ -421,8 +406,26 @@ goto error; #endif } - if (dircheck(self, nameobj) < 0) + + self->blksize = DEFAULT_BUFFER_SIZE; +#ifdef HAVE_FSTAT + if (fstat(self->fd, &fdfstat) < 0) goto error; +#if defined(S_ISDIR) && defined(EISDIR) + /* On Unix, open will succeed for directories. + In Python, there should be no file objects referring to + directories, so we need a check. */ + if (S_ISDIR(fdfstat.st_mode)) { + errno = EISDIR; + PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, nameobj); + goto error; + } +#endif /* defined(S_ISDIR) */ +#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE + if (fdfstat.st_blksize > 1) + self->blksize = fdfstat.st_blksize; +#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */ +#endif /* HAVE_FSTAT */ #if defined(MS_WINDOWS) || defined(__CYGWIN__) /* don't translate newlines (\r\n <=> \n) */ @@ -1216,6 +1219,7 @@ }; static PyMemberDef fileio_members[] = { + {"_blksize", T_UINT, offsetof(fileio, blksize), 0}, {"_finalizing", T_BOOL, offsetof(fileio, finalizing), 0}, {NULL} }; -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Merge 3.4.
by stefan.krah 29 Jun '14

29 Jun '14

http://hg.python.org/cpython/rev/04562803d346 changeset: 91469:04562803d346 parent: 91467:10cf594ace4b parent: 91468:49cdb04bfcc6 user: Stefan Krah <skrah(a)bytereef.org> date: Mon Jun 30 00:16:09 2014 +0200 summary: Merge 3.4. files: Doc/c-api/buffer.rst | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -489,8 +489,8 @@ :c:member:`view->obj` to *NULL* and return -1; If this function is used as part of a :ref:`getbufferproc <buffer-structs>`, - *exporter* MUST be set to the exporting object. Otherwise, *exporter* MUST - be NULL. + *exporter* MUST be set to the exporting object and *flags* must be passed + unmodified. Otherwise, *exporter* MUST be NULL. -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.4): Issue #21778: Clarify use of flags if PyBuffer_FillInfo() is used inside a
by stefan.krah 29 Jun '14

29 Jun '14

http://hg.python.org/cpython/rev/49cdb04bfcc6 changeset: 91468:49cdb04bfcc6 branch: 3.4 parent: 91463:0e08ca451b34 user: Stefan Krah <skrah(a)bytereef.org> date: Mon Jun 30 00:15:45 2014 +0200 summary: Issue #21778: Clarify use of flags if PyBuffer_FillInfo() is used inside a getbufferproc(). files: Doc/c-api/buffer.rst | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -489,8 +489,8 @@ :c:member:`view->obj` to *NULL* and return -1; If this function is used as part of a :ref:`getbufferproc <buffer-structs>`, - *exporter* MUST be set to the exporting object. Otherwise, *exporter* MUST - be NULL. + *exporter* MUST be set to the exporting object and *flags* must be passed + unmodified. Otherwise, *exporter* MUST be NULL. -- Repository URL: http://hg.python.org/cpython

1 0

cpython: Fixes #10541: regrtest -T is broken
by alexander.belopolsky 29 Jun '14

29 Jun '14

http://hg.python.org/cpython/rev/10cf594ace4b changeset: 91467:10cf594ace4b parent: 91464:394e6bda5a70 user: Alexander Belopolsky <alexander.belopolsky(a)gmail.com> date: Sun Jun 29 17:44:05 2014 -0400 summary: Fixes #10541: regrtest -T is broken * makes test_trace tests restore the tracefunc after they run * write_results() in trace module will not terminate if lnotab cannot be found. files: Lib/test/test_trace.py | 6 +++++- Lib/trace.py | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -10,7 +10,6 @@ from test.tracedmodules import testmod - #------------------------------- Utilities -----------------------------------# def fix_ext_py(filename): @@ -224,6 +223,11 @@ self.addCleanup(sys.settrace, sys.gettrace()) self.tracer = Trace(count=0, trace=0, countfuncs=1) self.filemod = my_file_and_modname() + self._saved_tracefunc = sys.gettrace() + + def tearDown(self): + if self._saved_tracefunc is not None: + sys.settrace(self._saved_tracefunc) def test_simple_caller(self): self.tracer.runfunc(traced_func_simple_caller, 1) diff --git a/Lib/trace.py b/Lib/trace.py --- a/Lib/trace.py +++ b/Lib/trace.py @@ -326,16 +326,17 @@ lnotab = _find_executable_linenos(filename) else: lnotab = {} + if lnotab: + source = linecache.getlines(filename) + coverpath = os.path.join(dir, modulename + ".cover") + with open(filename, 'rb') as fp: + encoding, _ = tokenize.detect_encoding(fp.readline) + n_hits, n_lines = self.write_results_file(coverpath, source, + lnotab, count, encoding) + if summary and n_lines: + percent = int(100 * n_hits / n_lines) + sums[modulename] = n_lines, percent, modulename, filename - source = linecache.getlines(filename) - coverpath = os.path.join(dir, modulename + ".cover") - with open(filename, 'rb') as fp: - encoding, _ = tokenize.detect_encoding(fp.readline) - n_hits, n_lines = self.write_results_file(coverpath, source, - lnotab, count, encoding) - if summary and n_lines: - percent = int(100 * n_hits / n_lines) - sums[modulename] = n_lines, percent, modulename, filename if summary and sums: print("lines cov% module (path)") -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): add a test for access errors from OpenKey (closes #21871)
by benjamin.peterson 29 Jun '14

29 Jun '14

http://hg.python.org/cpython/rev/ee33d61f5e4b changeset: 91466:ee33d61f5e4b branch: 2.7 user: Benjamin Peterson <benjamin(a)python.org> date: Sun Jun 29 13:02:12 2014 -0700 summary: add a test for access errors from OpenKey (closes #21871) Patch from Vladimir Iofik. files: Lib/test/test_mimetypes.py | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -144,6 +144,23 @@ finally: mimetypes._winreg = _winreg + def test_registry_read_error(self): + import _winreg + + class MockWinreg(object): + def OpenKey(self, key, name): + if key != _winreg.HKEY_CLASSES_ROOT: + raise WindowsError(5, "Access is denied") + return _winreg.OpenKey(key, name) + def __getattr__(self, name): + return getattr(_winreg, name) + + mimetypes._winreg = MockWinreg() + try: + mimetypes.init() + finally: + mimetypes._winreg = _winreg + def test_main(): test_support.run_unittest(MimeTypesTestCase, Win32MimeTypesTestCase -- Repository URL: http://hg.python.org/cpython

1 0

cpython (2.7): don't allow unicode into type_map on Windows (closes #21652)
by benjamin.peterson 29 Jun '14

29 Jun '14

http://hg.python.org/cpython/rev/1aafbdfba25a changeset: 91465:1aafbdfba25a branch: 2.7 parent: 91458:4ef517041573 user: Benjamin Peterson <benjamin(a)python.org> date: Sun Jun 29 12:58:16 2014 -0700 summary: don't allow unicode into type_map on Windows (closes #21652) Patch from Vladimir Iofik. files: Lib/mimetypes.py | 29 ++++++++++++++----------- Lib/test/test_mimetypes.py | 24 ++++++++++++++++++-- Misc/NEWS | 3 ++ 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -247,23 +247,26 @@ break i += 1 + default_encoding = sys.getdefaultencoding() with _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, '') as hkcr: for subkeyname in enum_types(hkcr): - # Only check file extensions, not all possible classes - if not subkeyname.startswith("."): - continue - - with _winreg.OpenKey(hkcr, subkeyname) as subkey: - # If there is no "Content Type" value, or if it is not - # a simple string, simply skip - try: + try: + with _winreg.OpenKey(hkcr, subkeyname) as subkey: + # Only check file extensions + if not subkeyname.startswith("."): + continue + # raises EnvironmentError if no 'Content Type' value mimetype, datatype = _winreg.QueryValueEx( subkey, 'Content Type') - except EnvironmentError: - continue - if datatype != _winreg.REG_SZ: - continue - self.add_type(mimetype, subkeyname, strict) + if datatype != _winreg.REG_SZ: + continue + try: + mimetype = mimetype.encode(default_encoding) + except UnicodeEncodeError: + continue + self.add_type(mimetype, subkeyname, strict) + except EnvironmentError: + continue def guess_type(url, strict=True): """Guess the type of a file based on its URL. diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import mimetypes import StringIO import unittest @@ -95,10 +97,10 @@ def __getattr__(self, name): if name == 'EnumKey': return lambda key, i: _winreg.EnumKey(key, i) + "\xa3" - elif name == "OpenKey": + elif name == 'OpenKey': return lambda key, name: _winreg.OpenKey(key, name.rstrip("\xa3")) elif name == 'QueryValueEx': - return lambda subkey, label: (label + "\xa3", _winreg.REG_SZ) + return lambda subkey, label: (u'текст/простой' , _winreg.REG_SZ) return getattr(_winreg, name) mimetypes._winreg = MockWinreg() @@ -115,7 +117,7 @@ class MockWinreg(object): def __getattr__(self, name): if name == 'QueryValueEx': - return lambda subkey, label: (label + "\xa3", _winreg.REG_SZ) + return lambda subkey, label: (u'текст/простой', _winreg.REG_SZ) return getattr(_winreg, name) mimetypes._winreg = MockWinreg() @@ -126,6 +128,22 @@ finally: mimetypes._winreg = _winreg + def test_type_map_values(self): + import _winreg + + class MockWinreg(object): + def __getattr__(self, name): + if name == 'QueryValueEx': + return lambda subkey, label: (u'text/plain', _winreg.REG_SZ) + return getattr(_winreg, name) + + mimetypes._winreg = MockWinreg() + try: + mimetypes.init() + self.assertTrue(isinstance(mimetypes.types_map.values()[0], str)) + finally: + mimetypes._winreg = _winreg + def test_main(): test_support.run_unittest(MimeTypesTestCase, Win32MimeTypesTestCase diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -35,6 +35,9 @@ Library ------- +- Issue #21652: Prevent mimetypes.type_map from containing unicode keys on + Windows. + - Issue #21729: Used the "with" statement in the dbm.dumb module to ensure files closing. -- Repository URL: http://hg.python.org/cpython

1 0

cpython (merge 3.4 -> default): Issue #20753: Merge with 3.4.
by berker.peksag 29 Jun '14

29 Jun '14

http://hg.python.org/cpython/rev/394e6bda5a70 changeset: 91464:394e6bda5a70 parent: 91462:4f3a8829c069 parent: 91463:0e08ca451b34 user: Berker Peksag <berker.peksag(a)gmail.com> date: Sun Jun 29 15:56:21 2014 +0300 summary: Issue #20753: Merge with 3.4. files: Lib/test/test_robotparser.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -4,8 +4,11 @@ from urllib.error import URLError, HTTPError from urllib.request import urlopen from test import support -import threading from http.server import BaseHTTPRequestHandler, HTTPServer +try: + import threading +except ImportError: + threading = None class RobotTestCase(unittest.TestCase): @@ -259,6 +262,7 @@ pass +(a)unittest.skipUnless(threading, 'threading required for this test') class PasswordProtectedSiteTestCase(unittest.TestCase): def setUp(self): -- Repository URL: http://hg.python.org/cpython

1 0

cpython (3.4): Issue #20753: Skip PasswordProtectedSiteTestCase when Python is built without
by berker.peksag 29 Jun '14

29 Jun '14

http://hg.python.org/cpython/rev/0e08ca451b34 changeset: 91463:0e08ca451b34 branch: 3.4 parent: 91461:8734e881c400 user: Berker Peksag <berker.peksag(a)gmail.com> date: Sun Jun 29 15:54:56 2014 +0300 summary: Issue #20753: Skip PasswordProtectedSiteTestCase when Python is built without threads. files: Lib/test/test_robotparser.py | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -4,8 +4,11 @@ from urllib.error import URLError, HTTPError from urllib.request import urlopen from test import support -import threading from http.server import BaseHTTPRequestHandler, HTTPServer +try: + import threading +except ImportError: + threading = None class RobotTestCase(unittest.TestCase): @@ -259,6 +262,7 @@ pass +(a)unittest.skipUnless(threading, 'threading required for this test') class PasswordProtectedSiteTestCase(unittest.TestCase): def setUp(self): -- Repository URL: http://hg.python.org/cpython

1 0

Cron <docs@dinsdale> /home/docs/build-devguide
by root＠python.org 29 Jun '14

29 Jun '14

abort: error: Connection timed out

1 0

Daily reference leaks (4f3a8829c069): sum=9
by solipsis＠pitrou.net 29 Jun '14

29 Jun '14

results for 4f3a8829c069 on branch "default" -------------------------------------------- test_collections leaked [0, -2, 2] references, sum=0 test_functools leaked [0, 0, 3] memory blocks, sum=3 test_io leaked [2, 2, 2] references, sum=6 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/antoine/cpython/refleaks/reflogNpeCa8', '-x']

1 0