Mailman 3 September 2010 - Python-checkins

r85121 - python/branches/py3k/Python/import.c
by brian.curtin 30 Sep '10

30 Sep '10

Author: brian.curtin Date: Wed Sep 29 21:09:33 2010 New Revision: 85121 Log: Remove an unreferenced variable. len is no longer needed. Modified: python/branches/py3k/Python/import.c Modified: python/branches/py3k/Python/import.c ============================================================================== --- python/branches/py3k/Python/import.c (original) +++ python/branches/py3k/Python/import.c Wed Sep 29 21:09:33 2010 @@ -3723,7 +3723,6 @@ PyObject *pathobj; DWORD rv; wchar_t *path; - Py_ssize_t len; if (!_PyArg_NoKeywords("NullImporter()", kwds)) return -1;

1 0

r85120 - in python/branches/py3k: Lib/test/test_math.py Modules/mathmodule.c
by mark.dickinson 30 Sep '10

30 Sep '10

Author: mark.dickinson Date: Wed Sep 29 21:06:36 2010 New Revision: 85120 Log: Issue #9599: Further accuracy tweaks to loghelper. For an integer n that's small enough to be converted to a float without overflow, log(n) is now computed as log(float(n)), and similarly for log10. Modified: python/branches/py3k/Lib/test/test_math.py python/branches/py3k/Modules/mathmodule.c Modified: python/branches/py3k/Lib/test/test_math.py ============================================================================== --- python/branches/py3k/Lib/test/test_math.py (original) +++ python/branches/py3k/Lib/test/test_math.py Wed Sep 29 21:06:36 2010 @@ -641,8 +641,12 @@ self.ftest('log(32,2)', math.log(32,2), 5) self.ftest('log(10**40, 10)', math.log(10**40, 10), 40) self.ftest('log(10**40, 10**20)', math.log(10**40, 10**20), 2) - self.assertEquals(math.log(INF), INF) + self.ftest('log(10**1000)', math.log(10**1000), + 2302.5850929940457) + self.assertRaises(ValueError, math.log, -1.5) + self.assertRaises(ValueError, math.log, -10**1000) self.assertRaises(ValueError, math.log, NINF) + self.assertEquals(math.log(INF), INF) self.assertTrue(math.isnan(math.log(NAN))) def testLog1p(self): @@ -655,8 +659,11 @@ self.ftest('log10(0.1)', math.log10(0.1), -1) self.ftest('log10(1)', math.log10(1), 0) self.ftest('log10(10)', math.log10(10), 1) - self.assertEquals(math.log(INF), INF) + self.ftest('log10(10**1000)', math.log10(10**1000), 1000.0) + self.assertRaises(ValueError, math.log10, -1.5) + self.assertRaises(ValueError, math.log10, -10**1000) self.assertRaises(ValueError, math.log10, NINF) + self.assertEquals(math.log(INF), INF) self.assertTrue(math.isnan(math.log10(NAN))) def testModf(self): Modified: python/branches/py3k/Modules/mathmodule.c ============================================================================== --- python/branches/py3k/Modules/mathmodule.c (original) +++ python/branches/py3k/Modules/mathmodule.c Wed Sep 29 21:06:36 2010 @@ -1562,25 +1562,33 @@ { /* If it is long, do it ourselves. */ if (PyLong_Check(arg)) { - double x; + double x, result; Py_ssize_t e; - x = _PyLong_Frexp((PyLongObject *)arg, &e); - if (x == -1.0 && PyErr_Occurred()) - return NULL; - if (x <= 0.0) { + + /* Negative or zero inputs give a ValueError. */ + if (Py_SIZE(arg) <= 0) { PyErr_SetString(PyExc_ValueError, "math domain error"); return NULL; } - /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. - It's slightly better to compute the log as log(2 * x) + log(2) * (e - - 1): then when 'arg' is a power of 2, 2**k say, this gives us 0.0 + - log(2) * k instead of log(0.5) + log(2)*(k+1), and so marginally - increases the chances of log(arg, 2) returning the correct result. - */ - x = func(2.0 * x) + func(2.0) * (e - 1); - return PyFloat_FromDouble(x); + x = PyLong_AsDouble(arg); + if (x == -1.0 && PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + /* Here the conversion to double overflowed, but it's possible + to compute the log anyway. Clear the exception and continue. */ + PyErr_Clear(); + x = _PyLong_Frexp((PyLongObject *)arg, &e); + if (x == -1.0 && PyErr_Occurred()) + return NULL; + /* Value is ~= x * 2**e, so the log ~= log(x) + log(2) * e. */ + result = func(x) + func(2.0) * e; + } + else + /* Successfully converted x to a double. */ + result = func(x); + return PyFloat_FromDouble(result); } /* Else let libm handle it by itself. */

1 0

r85119 - python/branches/py3k/Lib/test/formatfloat_testcases.txt
by mark.dickinson 30 Sep '10

30 Sep '10

Author: mark.dickinson Date: Wed Sep 29 20:41:54 2010 New Revision: 85119 Log: Add testcases from bug report #9980. Modified: python/branches/py3k/Lib/test/formatfloat_testcases.txt Modified: python/branches/py3k/Lib/test/formatfloat_testcases.txt ============================================================================== --- python/branches/py3k/Lib/test/formatfloat_testcases.txt (original) +++ python/branches/py3k/Lib/test/formatfloat_testcases.txt Wed Sep 29 20:41:54 2010 @@ -279,6 +279,11 @@ %.2g 0.000123 -> 0.00012 %.2g 0.0000123 -> 1.2e-05 +-- bad cases from http://bugs.python.org/issue9980 +%.12g 38210.0 -> 38210 +%.12g 37210.0 -> 37210 +%.12g 36210.0 -> 36210 + -- alternate g formatting: always include decimal point and -- exactly <precision> significant digits. %#.0g 0 -> 0.

1 0

r85118 - python/branches/py3k/Objects/unicodeobject.c
by victor.stinner 30 Sep '10

30 Sep '10

Author: victor.stinner Date: Wed Sep 29 19:55:12 2010 New Revision: 85118 Log: Fix PyUnicode_AsWideCharString(): set *size if size is not NULL Modified: python/branches/py3k/Objects/unicodeobject.c Modified: python/branches/py3k/Objects/unicodeobject.c ============================================================================== --- python/branches/py3k/Objects/unicodeobject.c (original) +++ python/branches/py3k/Objects/unicodeobject.c Wed Sep 29 19:55:12 2010 @@ -1215,6 +1215,8 @@ return NULL; } unicode_aswidechar(unicode, buffer, buflen); + if (size) + *size = buflen; return buffer; }

1 0

r85117 - python/branches/py3k/Lib/test/test_imp.py
by victor.stinner 30 Sep '10

30 Sep '10

Author: victor.stinner Date: Wed Sep 29 18:59:46 2010 New Revision: 85117 Log: test_imp: getfilesystemencoding() cannot be None anymore And the codec name is normalized. Modified: python/branches/py3k/Lib/test/test_imp.py Modified: python/branches/py3k/Lib/test/test_imp.py ============================================================================== --- python/branches/py3k/Lib/test/test_imp.py (original) +++ python/branches/py3k/Lib/test/test_imp.py Wed Sep 29 18:59:46 2010 @@ -87,7 +87,6 @@ # the return encoding could be uppercase or None fs_encoding = sys.getfilesystemencoding() - fs_encoding = fs_encoding.lower() if fs_encoding else 'ascii' # covers utf-8 and Windows ANSI code pages # one non-space symbol from every page

1 0

r85116 - in python/branches/py3k: Demo/zlib/minigzip.py Lib/posixpath.py
by victor.stinner 30 Sep '10

30 Sep '10

Author: victor.stinner Date: Wed Sep 29 18:59:18 2010 New Revision: 85116 Log: Use os.fsencode() to support surrogates Modified: python/branches/py3k/Demo/zlib/minigzip.py python/branches/py3k/Lib/posixpath.py Modified: python/branches/py3k/Demo/zlib/minigzip.py ============================================================================== --- python/branches/py3k/Demo/zlib/minigzip.py (original) +++ python/branches/py3k/Demo/zlib/minigzip.py Wed Sep 29 18:59:18 2010 @@ -31,7 +31,7 @@ write32(output, mtime) output.write(b'\002') # ... slowest compression alg. ... output.write(b'\377') # ... OS (=unknown) ... - bfilename = filename.encode(sys.getfilesystemencoding()) + bfilename = os.fsencode(filename) output.write(bfilename + b'\000') # ... original filename ... crcval = zlib.crc32(b'') Modified: python/branches/py3k/Lib/posixpath.py ============================================================================== --- python/branches/py3k/Lib/posixpath.py (original) +++ python/branches/py3k/Lib/posixpath.py Wed Sep 29 18:59:18 2010 @@ -262,7 +262,7 @@ return path userhome = pwent.pw_dir if isinstance(path, bytes): - userhome = userhome.encode(sys.getfilesystemencoding()) + userhome = os.fsencode(userhome) root = b'/' else: root = '/'

1 0

r85115 - in python/branches/py3k: Include/code.h Objects/codeobject.c Objects/object.c Objects/unicodeobject.c Python/pythonrun.c
by victor.stinner 30 Sep '10

30 Sep '10

Author: victor.stinner Date: Wed Sep 29 18:35:47 2010 New Revision: 85115 Log: Issue #9630: Redecode filenames when setting the filesystem encoding Redecode the filenames of: - all modules: __file__ and __path__ attributes - all code objects: co_filename attribute - sys.path - sys.meta_path - sys.executable - sys.path_importer_cache (keys) Keep weak references to all code objects until initfsencoding() is called, to be able to redecode co_filename attribute of all code objects. Modified: python/branches/py3k/Include/code.h python/branches/py3k/Objects/codeobject.c python/branches/py3k/Objects/object.c python/branches/py3k/Objects/unicodeobject.c python/branches/py3k/Python/pythonrun.c Modified: python/branches/py3k/Include/code.h ============================================================================== --- python/branches/py3k/Include/code.h (original) +++ python/branches/py3k/Include/code.h Wed Sep 29 18:35:47 2010 @@ -99,6 +99,13 @@ PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyObject *lineno_obj); +/* List of weak references to all code objects. The list is used by + initfsencoding() to redecode code filenames at startup if the filesystem + encoding changes. At initfsencoding() exit, the list is set to NULL and it + is no more used. */ + +extern PyObject *_Py_code_object_list; + #ifdef __cplusplus } #endif Modified: python/branches/py3k/Objects/codeobject.c ============================================================================== --- python/branches/py3k/Objects/codeobject.c (original) +++ python/branches/py3k/Objects/codeobject.c Wed Sep 29 18:35:47 2010 @@ -5,6 +5,8 @@ #define NAME_CHARS \ "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz" +PyObject *_Py_code_object_list = NULL; + /* all_name_chars(s): true iff all chars in s are valid NAME_CHARS */ static int @@ -109,8 +111,23 @@ co->co_lnotab = lnotab; co->co_zombieframe = NULL; co->co_weakreflist = NULL; + + if (_Py_code_object_list != NULL) { + int err; + PyObject *ref = PyWeakref_NewRef((PyObject*)co, NULL); + if (ref == NULL) + goto error; + err = PyList_Append(_Py_code_object_list, ref); + Py_DECREF(ref); + if (err) + goto error; + } } return co; + +error: + Py_DECREF(co); + return NULL; } PyCodeObject * Modified: python/branches/py3k/Objects/object.c ============================================================================== --- python/branches/py3k/Objects/object.c (original) +++ python/branches/py3k/Objects/object.c Wed Sep 29 18:35:47 2010 @@ -1604,6 +1604,10 @@ if (PyType_Ready(&PyCode_Type) < 0) Py_FatalError("Can't initialize code type"); + _Py_code_object_list = PyList_New(0); + if (_Py_code_object_list == NULL) + Py_FatalError("Can't initialize code type"); + if (PyType_Ready(&PyFrame_Type) < 0) Py_FatalError("Can't initialize frame type"); Modified: python/branches/py3k/Objects/unicodeobject.c ============================================================================== --- python/branches/py3k/Objects/unicodeobject.c (original) +++ python/branches/py3k/Objects/unicodeobject.c Wed Sep 29 18:35:47 2010 @@ -1510,10 +1510,14 @@ return PyUnicode_AsEncodedString(unicode, Py_FileSystemDefaultEncoding, "surrogateescape"); - } else + } + else { + /* if you change the default encoding, update also + PyUnicode_DecodeFSDefaultAndSize() and redecode_filenames() */ return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode), PyUnicode_GET_SIZE(unicode), "surrogateescape"); + } } PyObject *PyUnicode_AsEncodedString(PyObject *unicode, @@ -1680,6 +1684,8 @@ "surrogateescape"); } else { + /* if you change the default encoding, update also + PyUnicode_EncodeFSDefault() and redecode_filenames() */ return PyUnicode_DecodeUTF8(s, size, "surrogateescape"); } } Modified: python/branches/py3k/Python/pythonrun.c ============================================================================== --- python/branches/py3k/Python/pythonrun.c (original) +++ python/branches/py3k/Python/pythonrun.c Wed Sep 29 18:35:47 2010 @@ -719,6 +719,259 @@ } } +/* Redecode a filename from the default filesystem encoding (utf-8) to + 'new_encoding' encoding with 'errors' error handler */ +static PyObject* +redecode_filename(PyObject *file, const char *new_encoding, + const char *errors) +{ + PyObject *file_bytes = NULL, *new_file = NULL; + + file_bytes = PyUnicode_EncodeFSDefault(file); + if (file_bytes == NULL) + return NULL; + new_file = PyUnicode_Decode( + PyBytes_AsString(file_bytes), + PyBytes_GET_SIZE(file_bytes), + new_encoding, + errors); + Py_DECREF(file_bytes); + return new_file; +} + +/* Redecode a path list */ +static int +redecode_path_list(PyObject *paths, + const char *new_encoding, const char *errors) +{ + PyObject *filename, *new_filename; + Py_ssize_t i, size; + + size = PyList_Size(paths); + for (i=0; i < size; i++) { + filename = PyList_GetItem(paths, i); + if (filename == NULL) + return -1; + + new_filename = redecode_filename(filename, new_encoding, errors); + if (new_filename == NULL) + return -1; + if (PyList_SetItem(paths, i, new_filename)) { + Py_DECREF(new_filename); + return -1; + } + } + return 0; +} + +/* Redecode __file__ and __path__ attributes of sys.modules */ +static int +redecode_sys_modules(const char *new_encoding, const char *errors) +{ + PyInterpreterState *interp; + PyObject *modules, *values, *file, *new_file, *paths; + PyObject *iter = NULL, *module = NULL; + + interp = PyThreadState_GET()->interp; + modules = interp->modules; + + values = PyObject_CallMethod(modules, "values", ""); + if (values == NULL) + goto error; + + iter = PyObject_GetIter(values); + Py_DECREF(values); + if (iter == NULL) + goto error; + + while (1) + { + module = PyIter_Next(iter); + if (module == NULL) { + if (PyErr_Occurred()) + goto error; + else + break; + } + + file = PyModule_GetFilenameObject(module); + if (file != NULL) { + new_file = redecode_filename(file, new_encoding, errors); + Py_DECREF(file); + if (new_file == NULL) + goto error; + if (PyObject_SetAttrString(module, "__file__", new_file)) { + Py_DECREF(new_file); + goto error; + } + Py_DECREF(new_file); + } + else + PyErr_Clear(); + + paths = PyObject_GetAttrString(module, "__path__"); + if (paths != NULL) { + if (redecode_path_list(paths, new_encoding, errors)) + goto error; + } + else + PyErr_Clear(); + + Py_CLEAR(module); + } + Py_CLEAR(iter); + return 0; + +error: + Py_XDECREF(iter); + Py_XDECREF(module); + return -1; +} + +/* Redecode sys.path_importer_cache keys */ +static int +redecode_sys_path_importer_cache(const char *new_encoding, const char *errors) +{ + PyObject *path_importer_cache, *items, *item, *path, *importer, *new_path; + PyObject *new_cache = NULL, *iter = NULL; + + path_importer_cache = PySys_GetObject("path_importer_cache"); + if (path_importer_cache == NULL) + goto error; + + items = PyObject_CallMethod(path_importer_cache, "items", ""); + if (items == NULL) + goto error; + + iter = PyObject_GetIter(items); + Py_DECREF(items); + if (iter == NULL) + goto error; + + new_cache = PyDict_New(); + if (new_cache == NULL) + goto error; + + while (1) + { + item = PyIter_Next(iter); + if (item == NULL) { + if (PyErr_Occurred()) + goto error; + else + break; + } + path = PyTuple_GET_ITEM(item, 0); + importer = PyTuple_GET_ITEM(item, 1); + + new_path = redecode_filename(path, new_encoding, errors); + if (new_path == NULL) + goto error; + if (PyDict_SetItem(new_cache, new_path, importer)) { + Py_DECREF(new_path); + goto error; + } + Py_DECREF(new_path); + } + Py_CLEAR(iter); + if (PySys_SetObject("path_importer_cache", new_cache)) + goto error; + Py_CLEAR(new_cache); + return 0; + +error: + Py_XDECREF(iter); + Py_XDECREF(new_cache); + return -1; +} + +/* Redecode co_filename attribute of all code objects */ +static int +redecode_code_objects(const char *new_encoding, const char *errors) +{ + Py_ssize_t i, len; + PyCodeObject *co; + PyObject *ref, *new_file; + + len = Py_SIZE(_Py_code_object_list); + for (i=0; i < len; i++) { + ref = PyList_GET_ITEM(_Py_code_object_list, i); + co = (PyCodeObject *)PyWeakref_GetObject(ref); + if ((PyObject*)co == Py_None) + continue; + if (co == NULL) + return -1; + + new_file = redecode_filename(co->co_filename, new_encoding, errors); + if (new_file == NULL) + return -1; + Py_DECREF(co->co_filename); + co->co_filename = new_file; + } + Py_CLEAR(_Py_code_object_list); + return 0; +} + +/* Redecode the filenames of all modules (__file__ and __path__ attributes), + all code objects (co_filename attribute), sys.path, sys.meta_path, + sys.executable and sys.path_importer_cache (keys) when the filesystem + encoding changes from the default encoding (utf-8) to new_encoding */ +static int +redecode_filenames(const char *new_encoding) +{ + char *errors; + PyObject *paths, *executable, *new_executable; + + /* PyUnicode_DecodeFSDefault() and PyUnicode_EncodeFSDefault() do already + use utf-8 if Py_FileSystemDefaultEncoding is NULL */ + if (strcmp(new_encoding, "utf-8") == 0) + return 0; + + if (strcmp(new_encoding, "mbcs") != 0) + errors = "surrogateescape"; + else + errors = NULL; + + /* sys.modules */ + if (redecode_sys_modules(new_encoding, errors)) + return -1; + + /* sys.path and sys.meta_path */ + paths = PySys_GetObject("path"); + if (paths != NULL) { + if (redecode_path_list(paths, new_encoding, errors)) + return -1; + } + paths = PySys_GetObject("meta_path"); + if (paths != NULL) { + if (redecode_path_list(paths, new_encoding, errors)) + return -1; + } + + /* sys.executable */ + executable = PySys_GetObject("executable"); + if (executable == NULL) + return -1; + new_executable = redecode_filename(executable, new_encoding, errors); + if (new_executable == NULL) + return -1; + if (PySys_SetObject("executable", new_executable)) { + Py_DECREF(new_executable); + return -1; + } + Py_DECREF(new_executable); + + /* sys.path_importer_cache */ + if (redecode_sys_path_importer_cache(new_encoding, errors)) + return -1; + + /* code objects */ + if (redecode_code_objects(new_encoding, errors)) + return -1; + + return 0; +} + static void initfsencoding(void) { @@ -744,8 +997,11 @@ codeset = get_codeset(); } if (codeset != NULL) { + if (redecode_filenames(codeset)) + Py_FatalError("Py_Initialize: can't redecode filenames"); Py_FileSystemDefaultEncoding = codeset; Py_HasFileSystemDefaultEncoding = 0; + Py_CLEAR(_Py_code_object_list); return; } else { fprintf(stderr, "Unable to get the locale encoding:\n"); @@ -758,6 +1014,8 @@ } #endif + Py_CLEAR(_Py_code_object_list); + /* the encoding is mbcs, utf-8 or ascii */ codec = _PyCodec_Lookup(Py_FileSystemDefaultEncoding); if (!codec) {

1 0

r85114 - in python/branches/py3k/Lib: nntplib.py test/test_nntplib.py
by antoine.pitrou 30 Sep '10

30 Sep '10

Author: antoine.pitrou Date: Wed Sep 29 18:19:50 2010 New Revision: 85114 Log: Fix NNTP when there's a ".netrc" file Modified: python/branches/py3k/Lib/nntplib.py python/branches/py3k/Lib/test/test_nntplib.py Modified: python/branches/py3k/Lib/nntplib.py ============================================================================== --- python/branches/py3k/Lib/nntplib.py (original) +++ python/branches/py3k/Lib/nntplib.py Wed Sep 29 18:19:50 2010 @@ -279,11 +279,12 @@ encoding = 'utf-8' errors = 'surrogateescape' - def __init__(self, file, user=None, password=None, + def __init__(self, file, host, user=None, password=None, readermode=None, usenetrc=True, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Initialize an instance. Arguments: - file: file-like object (open for read/write in binary mode) + - host: hostname of the server (used if `usenetrc` is True) - user: username to authenticate with - password: password to use with username - readermode: if true, send 'mode reader' command after @@ -933,7 +934,7 @@ self.port = port self.sock = socket.create_connection((host, port), timeout) file = self.sock.makefile("rwb") - _NNTPBase.__init__(self, file, user, password, + _NNTPBase.__init__(self, file, host, user, password, readermode, usenetrc, timeout) def _close(self): Modified: python/branches/py3k/Lib/test/test_nntplib.py ============================================================================== --- python/branches/py3k/Lib/test/test_nntplib.py (original) +++ python/branches/py3k/Lib/test/test_nntplib.py Wed Sep 29 18:19:50 2010 @@ -12,6 +12,7 @@ # TODO: # - test the `file` arg to more commands # - test error conditions +# - test auth and `usenetrc` class NetworkedNNTPTestsMixin: @@ -255,7 +256,7 @@ # isn't seekable. file = io.BufferedRWPair(self.sio, self.sio) kwargs.setdefault('usenetrc', False) - self.server = nntplib._NNTPBase(file, *args, **kwargs) + self.server = nntplib._NNTPBase(file, 'test.server', *args, **kwargs) return self.server

1 0

r85113 - python/branches/py3k/Lib/test/test_nntplib.py
by antoine.pitrou 30 Sep '10

30 Sep '10

Author: antoine.pitrou Date: Wed Sep 29 18:08:29 2010 New Revision: 85113 Log: Disable the usenetrc option when testing nntplib Modified: python/branches/py3k/Lib/test/test_nntplib.py Modified: python/branches/py3k/Lib/test/test_nntplib.py ============================================================================== --- python/branches/py3k/Lib/test/test_nntplib.py (original) +++ python/branches/py3k/Lib/test/test_nntplib.py Wed Sep 29 18:08:29 2010 @@ -164,7 +164,7 @@ def setUp(self): support.requires("network") with support.transient_internet(self.NNTP_HOST): - self.server = NNTP(self.NNTP_HOST, timeout=TIMEOUT) + self.server = NNTP(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) def tearDown(self): if self.server is not None: @@ -254,6 +254,7 @@ # Using BufferedRWPair instead of BufferedRandom ensures the file # isn't seekable. file = io.BufferedRWPair(self.sio, self.sio) + kwargs.setdefault('usenetrc', False) self.server = nntplib._NNTPBase(file, *args, **kwargs) return self.server

1 0

r85112 - python/branches/py3k-issue9978
by hirokazu.yamamoto 30 Sep '10

30 Sep '10

Author: hirokazu.yamamoto Date: Wed Sep 29 17:50:39 2010 New Revision: 85112 Log: Remove branch. Removed: python/branches/py3k-issue9978/

1 0