Mailman 3 December 2015 - Python-checkins

Daily reference leaks (29ea3827cfaa): sum=4
by solipsis＠pitrou.net 22 Dec '15

22 Dec '15

results for 29ea3827cfaa on branch "default" -------------------------------------------- test_functools leaked [0, 2, 2] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogxRhuKr', '--timeout', '7200']

1 0

cpython (3.5): Issue #25914: Fixed and simplified OrderedDict.__sizeof__.
by serhiy.storchaka 21 Dec '15

21 Dec '15

https://hg.python.org/cpython/rev/0a2ea08fcebe changeset: 99660:0a2ea08fcebe branch: 3.5 parent: 99658:c5e419464585 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Tue Dec 22 08:16:18 2015 +0200 summary: Issue #25914: Fixed and simplified OrderedDict.__sizeof__. files: Include/dictobject.h | 2 +- Lib/test/test_ordered_dict.py | 32 +++++++++++++++++++++++ Misc/NEWS | 2 + Objects/dictobject.c | 12 ++++++-- Objects/odictobject.c | 22 +--------------- 5 files changed, 45 insertions(+), 25 deletions(-) diff --git a/Include/dictobject.h b/Include/dictobject.h --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -98,7 +98,7 @@ PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp); PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); Py_ssize_t _PyDict_KeysSize(PyDictKeysObject *keys); -PyObject *_PyDict_SizeOf(PyDictObject *); +Py_ssize_t _PyDict_SizeOf(PyDictObject *); PyObject *_PyDict_Pop(PyDictObject *, PyObject *, PyObject *); PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -2,6 +2,7 @@ import copy import pickle from random import randrange, shuffle +import struct import sys import unittest from collections.abc import MutableMapping @@ -596,6 +597,37 @@ module = c_coll OrderedDict = c_coll.OrderedDict + check_sizeof = support.check_sizeof + + @support.cpython_only + def test_sizeof_exact(self): + OrderedDict = self.OrderedDict + calcsize = struct.calcsize + size = support.calcobjsize + check = self.check_sizeof + + basicsize = size('n2P' + '3PnPn2P') + calcsize('2nPn') + entrysize = calcsize('n2P') + calcsize('P') + nodesize = calcsize('Pn2P') + + od = OrderedDict() + check(od, basicsize + 8*entrysize) + od.x = 1 + check(od, basicsize + 8*entrysize) + od.update([(i, i) for i in range(3)]) + check(od, basicsize + 8*entrysize + 3*nodesize) + od.update([(i, i) for i in range(3, 10)]) + check(od, basicsize + 16*entrysize + 10*nodesize) + + check(od.keys(), size('P')) + check(od.items(), size('P')) + check(od.values(), size('P')) + + itersize = size('iP2n2P') + check(iter(od), itersize) + check(iter(od.keys()), itersize) + check(iter(od.items()), itersize) + check(iter(od.values()), itersize) def test_key_change_during_iteration(self): OrderedDict = self.OrderedDict diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,8 @@ Library ------- +- Issue #25914: Fixed and simplified OrderedDict.__sizeof__. + - Issue #25902: Fixed various refcount issues in ElementTree iteration. - Issue #25717: Restore the previous behaviour of tolerating most fstat() diff --git a/Objects/dictobject.c b/Objects/dictobject.c --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2554,7 +2554,7 @@ static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -PyObject * +Py_ssize_t _PyDict_SizeOf(PyDictObject *mp) { Py_ssize_t size, res; @@ -2567,7 +2567,7 @@ in the type object. */ if (mp->ma_keys->dk_refcnt == 1) res += sizeof(PyDictKeysObject) + (size-1) * sizeof(PyDictKeyEntry); - return PyLong_FromSsize_t(res); + return res; } Py_ssize_t @@ -2576,6 +2576,12 @@ return sizeof(PyDictKeysObject) + (DK_SIZE(keys)-1) * sizeof(PyDictKeyEntry); } +PyObject * +dict_sizeof(PyDictObject *mp) +{ + return PyLong_FromSsize_t(_PyDict_SizeOf(mp)); +} + PyDoc_STRVAR(getitem__doc__, "x.__getitem__(y) <==> x[y]"); PyDoc_STRVAR(sizeof__doc__, @@ -2623,7 +2629,7 @@ DICT___CONTAINS___METHODDEF {"__getitem__", (PyCFunction)dict_subscript, METH_O | METH_COEXIST, getitem__doc__}, - {"__sizeof__", (PyCFunction)_PyDict_SizeOf, METH_NOARGS, + {"__sizeof__", (PyCFunction)dict_sizeof, METH_NOARGS, sizeof__doc__}, {"get", (PyCFunction)dict_get, METH_VARARGS, get__doc__}, diff --git a/Objects/odictobject.c b/Objects/odictobject.c --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -940,27 +940,7 @@ static PyObject * odict_sizeof(PyODictObject *od) { - PyObject *pylong; - Py_ssize_t res, temp; - - pylong = _PyDict_SizeOf((PyDictObject *)od); - if (pylong == NULL) - return NULL; - res = PyLong_AsSsize_t(pylong); - Py_DECREF(pylong); - if (res == -1 && PyErr_Occurred()) - return NULL; - - /* instance dict */ - pylong = _PyDict_SizeOf((PyDictObject *)od->od_inst_dict); - if (pylong == NULL) - return NULL; - temp = PyLong_AsSsize_t(pylong); - Py_DECREF(pylong); - if (temp == -1 && PyErr_Occurred()) - return NULL; - res += temp; - + Py_ssize_t res = _PyDict_SizeOf((PyDictObject *)od); res += sizeof(_ODictNode *) * _odict_FAST_SIZE(od); /* od_fast_nodes */ if (!_odict_EMPTY(od)) { res += sizeof(_ODictNode) * PyODict_SIZE(od); /* linked-list */ -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.5 -> default): Issue #25914: Fixed and simplified OrderedDict.__sizeof__.
by serhiy.storchaka 21 Dec '15

21 Dec '15

https://hg.python.org/cpython/rev/624316fbb39d changeset: 99661:624316fbb39d parent: 99659:29ea3827cfaa parent: 99660:0a2ea08fcebe user: Serhiy Storchaka <storchaka(a)gmail.com> date: Tue Dec 22 08:22:05 2015 +0200 summary: Issue #25914: Fixed and simplified OrderedDict.__sizeof__. files: Include/dictobject.h | 2 +- Lib/test/test_ordered_dict.py | 32 +++++++++++++++++++++++ Misc/NEWS | 2 + Objects/dictobject.c | 12 ++++++-- Objects/odictobject.c | 22 +--------------- 5 files changed, 45 insertions(+), 25 deletions(-) diff --git a/Include/dictobject.h b/Include/dictobject.h --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -98,7 +98,7 @@ PyAPI_FUNC(void) _PyDict_MaybeUntrack(PyObject *mp); PyAPI_FUNC(int) _PyDict_HasOnlyStringKeys(PyObject *mp); Py_ssize_t _PyDict_KeysSize(PyDictKeysObject *keys); -PyObject *_PyDict_SizeOf(PyDictObject *); +Py_ssize_t _PyDict_SizeOf(PyDictObject *); PyObject *_PyDict_Pop(PyDictObject *, PyObject *, PyObject *); PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -2,6 +2,7 @@ import copy import pickle from random import randrange, shuffle +import struct import sys import unittest from collections.abc import MutableMapping @@ -604,6 +605,37 @@ module = c_coll OrderedDict = c_coll.OrderedDict + check_sizeof = support.check_sizeof + + @support.cpython_only + def test_sizeof_exact(self): + OrderedDict = self.OrderedDict + calcsize = struct.calcsize + size = support.calcobjsize + check = self.check_sizeof + + basicsize = size('n2P' + '3PnPn2P') + calcsize('2nPn') + entrysize = calcsize('n2P') + calcsize('P') + nodesize = calcsize('Pn2P') + + od = OrderedDict() + check(od, basicsize + 8*entrysize) + od.x = 1 + check(od, basicsize + 8*entrysize) + od.update([(i, i) for i in range(3)]) + check(od, basicsize + 8*entrysize + 3*nodesize) + od.update([(i, i) for i in range(3, 10)]) + check(od, basicsize + 16*entrysize + 10*nodesize) + + check(od.keys(), size('P')) + check(od.items(), size('P')) + check(od.values(), size('P')) + + itersize = size('iP2n2P') + check(iter(od), itersize) + check(iter(od.keys()), itersize) + check(iter(od.items()), itersize) + check(iter(od.values()), itersize) def test_key_change_during_iteration(self): OrderedDict = self.OrderedDict diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,8 @@ Library ------- +- Issue #25914: Fixed and simplified OrderedDict.__sizeof__. + - Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster. - Issue #25873: Optimized iterating ElementTree. Iterating elements diff --git a/Objects/dictobject.c b/Objects/dictobject.c --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2557,7 +2557,7 @@ static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -PyObject * +Py_ssize_t _PyDict_SizeOf(PyDictObject *mp) { Py_ssize_t size, res; @@ -2570,7 +2570,7 @@ in the type object. */ if (mp->ma_keys->dk_refcnt == 1) res += sizeof(PyDictKeysObject) + (size-1) * sizeof(PyDictKeyEntry); - return PyLong_FromSsize_t(res); + return res; } Py_ssize_t @@ -2579,6 +2579,12 @@ return sizeof(PyDictKeysObject) + (DK_SIZE(keys)-1) * sizeof(PyDictKeyEntry); } +PyObject * +dict_sizeof(PyDictObject *mp) +{ + return PyLong_FromSsize_t(_PyDict_SizeOf(mp)); +} + PyDoc_STRVAR(getitem__doc__, "x.__getitem__(y) <==> x[y]"); PyDoc_STRVAR(sizeof__doc__, @@ -2626,7 +2632,7 @@ DICT___CONTAINS___METHODDEF {"__getitem__", (PyCFunction)dict_subscript, METH_O | METH_COEXIST, getitem__doc__}, - {"__sizeof__", (PyCFunction)_PyDict_SizeOf, METH_NOARGS, + {"__sizeof__", (PyCFunction)dict_sizeof, METH_NOARGS, sizeof__doc__}, {"get", (PyCFunction)dict_get, METH_VARARGS, get__doc__}, diff --git a/Objects/odictobject.c b/Objects/odictobject.c --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -940,27 +940,7 @@ static PyObject * odict_sizeof(PyODictObject *od) { - PyObject *pylong; - Py_ssize_t res, temp; - - pylong = _PyDict_SizeOf((PyDictObject *)od); - if (pylong == NULL) - return NULL; - res = PyLong_AsSsize_t(pylong); - Py_DECREF(pylong); - if (res == -1 && PyErr_Occurred()) - return NULL; - - /* instance dict */ - pylong = _PyDict_SizeOf((PyDictObject *)od->od_inst_dict); - if (pylong == NULL) - return NULL; - temp = PyLong_AsSsize_t(pylong); - Py_DECREF(pylong); - if (temp == -1 && PyErr_Occurred()) - return NULL; - res += temp; - + Py_ssize_t res = _PyDict_SizeOf((PyDictObject *)od); res += sizeof(_ODictNode *) * _odict_FAST_SIZE(od); /* od_fast_nodes */ if (!_odict_EMPTY(od)) { res += sizeof(_ODictNode) * PyODict_SIZE(od); /* linked-list */ -- Repository URL: https://hg.python.org/cpython

1 0

cpython (merge 3.5 -> default): Issue #25827: Merge with 3.5
by zach.ware 21 Dec '15

21 Dec '15

https://hg.python.org/cpython/rev/29ea3827cfaa changeset: 99659:29ea3827cfaa parent: 99656:090c3e3a648d parent: 99658:c5e419464585 user: Zachary Ware <zachary.ware(a)gmail.com> date: Mon Dec 21 14:17:12 2015 -0600 summary: Issue #25827: Merge with 3.5 files: .gitignore | 1 + .hgignore | 1 + Makefile.pre.in | 1 + Misc/NEWS | 3 + configure | 151 ++++++++++++++++++++++++++++++++++++ configure.ac | 39 +++++++++ 6 files changed, 196 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ *.gc?? *.profclang? *.profraw +*.dyn .gdb_history Doc/build/ Doc/venv/ diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -53,6 +53,7 @@ *.gc?? *.profclang? *.profraw +*.dyn Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff --git a/Makefile.pre.in b/Makefile.pre.in --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1593,6 +1593,7 @@ profile-removal: find . -name '*.gc??' -exec rm -f {} ';' find . -name '*.profclang?' -exec rm -f {} ';' + find . -name '*.dyn' -exec rm -f {} ';' rm -f $(COVERAGE_INFO) rm -rf $(COVERAGE_REPORT) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -509,6 +509,9 @@ Build ----- +- Issue #25827: Add support for building with ICC to ``configure``, including + a new ``--with-icc`` flag. + - Issue #25696: Fix installation of Python on UNIX with make -j9. - Issue #24986: It is now possible to build Python on Windows without errors diff --git a/configure b/configure --- a/configure +++ b/configure @@ -801,6 +801,7 @@ with_framework_name enable_framework with_gcc +with_icc with_cxx_main with_suffix enable_shared @@ -1480,6 +1481,7 @@ specify an alternate name of the framework built with --enable-framework --without-gcc never use gcc + --with-icc build with icc --with-cxx-main=<compiler> compile main() and link python executable with C++ compiler @@ -3549,6 +3551,29 @@ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $without_gcc" >&5 $as_echo "$without_gcc" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-icc" >&5 +$as_echo_n "checking for --with-icc... " >&6; } + +# Check whether --with-icc was given. +if test "${with_icc+set}" = set; then : + withval=$with_icc; + case $withval in + no) CC=${CC:-cc} + with_icc=no;; + yes) CC=icc + CXX=icpc + with_icc=yes;; + *) CC=$withval + with_icc=$withval;; + esac +else + + with_icc=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_icc" >&5 +$as_echo "$with_icc" >&6; } + # If the user switches compilers, we can't believe the cache if test ! -z "$ac_cv_prog_CC" -a ! -z "$CC" -a "$CC" != "$ac_cv_prog_CC" then @@ -4941,6 +4966,104 @@ CXX="$ac_cv_path_CXX" fi ;; + icc|*/icc) if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}icpc", so it can be a program name with args. +set dummy ${ac_tool_prefix}icpc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $CXX in + [\\/]* | ?:[\\/]*) + ac_cv_path_CXX="$CXX" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in notfound +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_CXX="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +CXX=$ac_cv_path_CXX +if test -n "$CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 +$as_echo "$CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_CXX"; then + ac_pt_CXX=$CXX + # Extract the first word of "icpc", so it can be a program name with args. +set dummy icpc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_CXX in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in notfound +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_CXX="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_CXX=$ac_cv_path_ac_pt_CXX +if test -n "$ac_pt_CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_CXX" >&5 +$as_echo "$ac_pt_CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_CXX" = x; then + CXX="icpc" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CXX=$ac_pt_CXX + fi +else + CXX="$ac_cv_path_CXX" +fi + ;; esac if test "$CXX" = "notfound" then @@ -6438,6 +6561,12 @@ # Enable PGO flags. + + + + + + # Extract the first word of "llvm-profdata", so it can be a program name with args. set dummy llvm-profdata; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -6509,6 +6638,12 @@ ;; esac ;; + *icc*) + PGO_PROF_GEN_FLAG="-prof-gen" + PGO_PROF_USE_FLAG="-prof-use" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; esac # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be @@ -6655,6 +6790,13 @@ BASECFLAGS="$BASECFLAGS -fno-strict-aliasing" fi + # ICC doesn't recognize the option, but only emits a warning + ## XXX does it emit an unused result warning and can it be disabled? + case "$CC" in + *icc*) + ac_cv_disable_unused_result_warning=no + ;; + *) { $as_echo "$as_me:${as_lineno-$LINENO}: checking if we can turn off $CC unused result warning" >&5 $as_echo_n "checking if we can turn off $CC unused result warning... " >&6; } ac_save_cc="$CC" @@ -6692,6 +6834,8 @@ CC="$ac_save_cc" { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_result_warning" >&5 $as_echo "$ac_cv_disable_unused_result_warning" >&6; } + ;; + esac if test $ac_cv_disable_unused_result_warning = yes then @@ -6981,6 +7125,13 @@ ;; esac +# ICC needs -fp-model strict or floats behave badly +case "$CC" in +*icc*) + CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" + ;; +esac + if test "$Py_DEBUG" = 'true'; then : else diff --git a/configure.ac b/configure.ac --- a/configure.ac +++ b/configure.ac @@ -593,6 +593,22 @@ esac]) AC_MSG_RESULT($without_gcc) +AC_MSG_CHECKING(for --with-icc) +AC_ARG_WITH(icc, + AS_HELP_STRING([--with-icc], [build with icc]), +[ + case $withval in + no) CC=${CC:-cc} + with_icc=no;; + yes) CC=icc + CXX=icpc + with_icc=yes;; + *) CC=$withval + with_icc=$withval;; + esac], [ + with_icc=no]) +AC_MSG_RESULT($with_icc) + # If the user switches compilers, we can't believe the cache if test ! -z "$ac_cv_prog_CC" -a ! -z "$CC" -a "$CC" != "$ac_cv_prog_CC" then @@ -699,6 +715,7 @@ gcc) AC_PATH_TOOL(CXX, [g++], [g++], [notfound]) ;; cc) AC_PATH_TOOL(CXX, [c++], [c++], [notfound]) ;; clang|*/clang) AC_PATH_TOOL(CXX, [clang++], [clang++], [notfound]) ;; + icc|*/icc) AC_PATH_TOOL(CXX, [icpc], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then @@ -1259,6 +1276,12 @@ ;; esac ;; + *icc*) + PGO_PROF_GEN_FLAG="-prof-gen" + PGO_PROF_USE_FLAG="-prof-use" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; esac # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be @@ -1370,6 +1393,13 @@ BASECFLAGS="$BASECFLAGS -fno-strict-aliasing" fi + # ICC doesn't recognize the option, but only emits a warning + ## XXX does it emit an unused result warning and can it be disabled? + case "$CC" in + *icc*) + ac_cv_disable_unused_result_warning=no + ;; + *) AC_MSG_CHECKING(if we can turn off $CC unused result warning) ac_save_cc="$CC" CC="$CC -Wunused-result -Werror" @@ -1386,6 +1416,8 @@ CFLAGS="$save_CFLAGS" CC="$ac_save_cc" AC_MSG_RESULT($ac_cv_disable_unused_result_warning) + ;; + esac if test $ac_cv_disable_unused_result_warning = yes then @@ -1608,6 +1640,13 @@ ;; esac +# ICC needs -fp-model strict or floats behave badly +case "$CC" in +*icc*) + CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" + ;; +esac + if test "$Py_DEBUG" = 'true'; then : else -- Repository URL: https://hg.python.org/cpython

1 0

cpython (3.5): Issue #25827: Add support for ICC to configure
by zach.ware 21 Dec '15

21 Dec '15

https://hg.python.org/cpython/rev/c5e419464585 changeset: 99658:c5e419464585 branch: 3.5 parent: 99653:00b6a13cfd70 user: Zachary Ware <zachary.ware(a)gmail.com> date: Mon Dec 21 12:09:17 2015 -0600 summary: Issue #25827: Add support for ICC to configure files: .gitignore | 1 + .hgignore | 1 + Makefile.pre.in | 1 + Misc/NEWS | 3 + configure | 151 ++++++++++++++++++++++++++++++++++++ configure.ac | 39 +++++++++ 6 files changed, 196 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ *.gc?? *.profclang? *.profraw +*.dyn .gdb_history Doc/build/ Doc/venv/ diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -53,6 +53,7 @@ *.gc?? *.profclang? *.profraw +*.dyn Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff --git a/Makefile.pre.in b/Makefile.pre.in --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1592,6 +1592,7 @@ profile-removal: find . -name '*.gc??' -exec rm -f {} ';' find . -name '*.profclang?' -exec rm -f {} ';' + find . -name '*.dyn' -exec rm -f {} ';' rm -f $(COVERAGE_INFO) rm -rf $(COVERAGE_REPORT) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -92,6 +92,9 @@ Build ----- +- Issue #25827: Add support for building with ICC to ``configure``, including + a new ``--with-icc`` flag. + - Issue #25696: Fix installation of Python on UNIX with make -j9. - Issue #25798: Update OS X 10.5 installer to use OpenSSL 1.0.2e. diff --git a/configure b/configure --- a/configure +++ b/configure @@ -801,6 +801,7 @@ with_framework_name enable_framework with_gcc +with_icc with_cxx_main with_suffix enable_shared @@ -1480,6 +1481,7 @@ specify an alternate name of the framework built with --enable-framework --without-gcc never use gcc + --with-icc build with icc --with-cxx-main=<compiler> compile main() and link python executable with C++ compiler @@ -3549,6 +3551,29 @@ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $without_gcc" >&5 $as_echo "$without_gcc" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-icc" >&5 +$as_echo_n "checking for --with-icc... " >&6; } + +# Check whether --with-icc was given. +if test "${with_icc+set}" = set; then : + withval=$with_icc; + case $withval in + no) CC=${CC:-cc} + with_icc=no;; + yes) CC=icc + CXX=icpc + with_icc=yes;; + *) CC=$withval + with_icc=$withval;; + esac +else + + with_icc=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_icc" >&5 +$as_echo "$with_icc" >&6; } + # If the user switches compilers, we can't believe the cache if test ! -z "$ac_cv_prog_CC" -a ! -z "$CC" -a "$CC" != "$ac_cv_prog_CC" then @@ -4941,6 +4966,104 @@ CXX="$ac_cv_path_CXX" fi ;; + icc|*/icc) if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}icpc", so it can be a program name with args. +set dummy ${ac_tool_prefix}icpc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $CXX in + [\\/]* | ?:[\\/]*) + ac_cv_path_CXX="$CXX" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in notfound +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_CXX="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +CXX=$ac_cv_path_CXX +if test -n "$CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 +$as_echo "$CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_CXX"; then + ac_pt_CXX=$CXX + # Extract the first word of "icpc", so it can be a program name with args. +set dummy icpc; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_ac_pt_CXX+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $ac_pt_CXX in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in notfound +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_CXX="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +ac_pt_CXX=$ac_cv_path_ac_pt_CXX +if test -n "$ac_pt_CXX"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_pt_CXX" >&5 +$as_echo "$ac_pt_CXX" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + if test "x$ac_pt_CXX" = x; then + CXX="icpc" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +$as_echo "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CXX=$ac_pt_CXX + fi +else + CXX="$ac_cv_path_CXX" +fi + ;; esac if test "$CXX" = "notfound" then @@ -6438,6 +6561,12 @@ # Enable PGO flags. + + + + + + # Extract the first word of "llvm-profdata", so it can be a program name with args. set dummy llvm-profdata; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -6509,6 +6638,12 @@ ;; esac ;; + *icc*) + PGO_PROF_GEN_FLAG="-prof-gen" + PGO_PROF_USE_FLAG="-prof-use" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; esac # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be @@ -6655,6 +6790,13 @@ BASECFLAGS="$BASECFLAGS -fno-strict-aliasing" fi + # ICC doesn't recognize the option, but only emits a warning + ## XXX does it emit an unused result warning and can it be disabled? + case "$CC" in + *icc*) + ac_cv_disable_unused_result_warning=no + ;; + *) { $as_echo "$as_me:${as_lineno-$LINENO}: checking if we can turn off $CC unused result warning" >&5 $as_echo_n "checking if we can turn off $CC unused result warning... " >&6; } ac_save_cc="$CC" @@ -6692,6 +6834,8 @@ CC="$ac_save_cc" { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_disable_unused_result_warning" >&5 $as_echo "$ac_cv_disable_unused_result_warning" >&6; } + ;; + esac if test $ac_cv_disable_unused_result_warning = yes then @@ -6981,6 +7125,13 @@ ;; esac +# ICC needs -fp-model strict or floats behave badly +case "$CC" in +*icc*) + CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" + ;; +esac + if test "$Py_DEBUG" = 'true'; then : else diff --git a/configure.ac b/configure.ac --- a/configure.ac +++ b/configure.ac @@ -593,6 +593,22 @@ esac]) AC_MSG_RESULT($without_gcc) +AC_MSG_CHECKING(for --with-icc) +AC_ARG_WITH(icc, + AS_HELP_STRING([--with-icc], [build with icc]), +[ + case $withval in + no) CC=${CC:-cc} + with_icc=no;; + yes) CC=icc + CXX=icpc + with_icc=yes;; + *) CC=$withval + with_icc=$withval;; + esac], [ + with_icc=no]) +AC_MSG_RESULT($with_icc) + # If the user switches compilers, we can't believe the cache if test ! -z "$ac_cv_prog_CC" -a ! -z "$CC" -a "$CC" != "$ac_cv_prog_CC" then @@ -699,6 +715,7 @@ gcc) AC_PATH_TOOL(CXX, [g++], [g++], [notfound]) ;; cc) AC_PATH_TOOL(CXX, [c++], [c++], [notfound]) ;; clang|*/clang) AC_PATH_TOOL(CXX, [clang++], [clang++], [notfound]) ;; + icc|*/icc) AC_PATH_TOOL(CXX, [icpc], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then @@ -1259,6 +1276,12 @@ ;; esac ;; + *icc*) + PGO_PROF_GEN_FLAG="-prof-gen" + PGO_PROF_USE_FLAG="-prof-use" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; esac # XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be @@ -1370,6 +1393,13 @@ BASECFLAGS="$BASECFLAGS -fno-strict-aliasing" fi + # ICC doesn't recognize the option, but only emits a warning + ## XXX does it emit an unused result warning and can it be disabled? + case "$CC" in + *icc*) + ac_cv_disable_unused_result_warning=no + ;; + *) AC_MSG_CHECKING(if we can turn off $CC unused result warning) ac_save_cc="$CC" CC="$CC -Wunused-result -Werror" @@ -1386,6 +1416,8 @@ CFLAGS="$save_CFLAGS" CC="$ac_save_cc" AC_MSG_RESULT($ac_cv_disable_unused_result_warning) + ;; + esac if test $ac_cv_disable_unused_result_warning = yes then @@ -1608,6 +1640,13 @@ ;; esac +# ICC needs -fp-model strict or floats behave badly +case "$CC" in +*icc*) + CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" + ;; +esac + if test "$Py_DEBUG" = 'true'; then : else -- Repository URL: https://hg.python.org/cpython

1 0

cpython (2.7): Issue #25827: Add support for ICC to configure
by zach.ware 21 Dec '15

21 Dec '15

https://hg.python.org/cpython/rev/d7b5c2f99a99 changeset: 99657:d7b5c2f99a99 branch: 2.7 parent: 99642:b87fe3e4a3d8 user: Zachary Ware <zachary.ware(a)gmail.com> date: Mon Dec 21 11:43:03 2015 -0600 summary: Issue #25827: Add support for ICC to configure files: .gitignore | 1 + .hgignore | 1 + Makefile.pre.in | 1 + Misc/NEWS | 3 ++ configure | 44 +++++++++++++++++++++++++++++++++++++ configure.ac | 29 ++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 0 deletions(-) diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.gc?? *.profclang? *.profraw +*.dyn Doc/build/ Doc/tools/docutils/ Doc/tools/jinja2/ diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -48,6 +48,7 @@ *.gc?? *.profclang? *.profraw +*.dyn Lib/distutils/command/*.pdb Lib/lib2to3/*.pickle Lib/test/data/* diff --git a/Makefile.pre.in b/Makefile.pre.in --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1354,6 +1354,7 @@ profile-removal: find . -name '*.gc??' -exec rm -f {} ';' find . -name '*.profclang?' -exec rm -f {} ';' + find . -name '*.dyn' -exec rm -f {} ';' clobber: clean profile-removal -rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -64,6 +64,9 @@ Build ----- +- Issue #25827: Add support for building with ICC to ``configure``, including + a new ``--with-icc`` flag. + - Issue #25696: Fix installation of Python on UNIX with make -j9. - Issue #25798: Update OS X 10.5+ 32-bit-only installer to build diff --git a/configure b/configure --- a/configure +++ b/configure @@ -789,6 +789,7 @@ with_framework_name enable_framework with_gcc +with_icc with_cxx_main with_suffix enable_shared @@ -1469,6 +1470,7 @@ specify an alternate name of the framework built with --enable-framework --without-gcc never use gcc + --with-icc build with icc --with-cxx-main=<compiler> compile main() and link python executable with C++ compiler @@ -3480,6 +3482,29 @@ { $as_echo "$as_me:${as_lineno-$LINENO}: result: $without_gcc" >&5 $as_echo "$without_gcc" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-icc" >&5 +$as_echo_n "checking for --with-icc... " >&6; } + +# Check whether --with-icc was given. +if test "${with_icc+set}" = set; then : + withval=$with_icc; + case $withval in + no) CC=${CC:-cc} + with_icc=no;; + yes) CC=icc + CXX=icpc + with_icc=yes;; + *) CC=$withval + with_icc=$withval;; + esac +else + + with_icc=no +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_icc" >&5 +$as_echo "$with_icc" >&6; } + # If the user switches compilers, we can't believe the cache if test ! -z "$ac_cv_prog_CC" -a ! -z "$CC" -a "$CC" != "$ac_cv_prog_CC" then @@ -6188,6 +6213,13 @@ ;; esac +# ICC needs -fp-model strict or floats behave badly +case "$CC" in +*icc*) + BASECFLAGS="$BASECFLAGS -fp-model strict" + ;; +esac + if test "$Py_DEBUG" = 'true'; then : else @@ -6334,6 +6366,12 @@ # Enable PGO flags. + + + + + + # Extract the first word of "llvm-profdata", so it can be a program name with args. set dummy llvm-profdata; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -6405,6 +6443,12 @@ ;; esac ;; + *icc*) + PGO_PROF_GEN_FLAG="-prof-gen" + PGO_PROF_USE_FLAG="-prof-use" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; esac diff --git a/configure.ac b/configure.ac --- a/configure.ac +++ b/configure.ac @@ -573,6 +573,22 @@ esac]) AC_MSG_RESULT($without_gcc) +AC_MSG_CHECKING(for --with-icc) +AC_ARG_WITH(icc, + AS_HELP_STRING([--with-icc], [build with icc]), +[ + case $withval in + no) CC=${CC:-cc} + with_icc=no;; + yes) CC=icc + CXX=icpc + with_icc=yes;; + *) CC=$withval + with_icc=$withval;; + esac], [ + with_icc=no]) +AC_MSG_RESULT($with_icc) + # If the user switches compilers, we can't believe the cache if test ! -z "$ac_cv_prog_CC" -a ! -z "$CC" -a "$CC" != "$ac_cv_prog_CC" then @@ -1266,6 +1282,13 @@ ;; esac +# ICC needs -fp-model strict or floats behave badly +case "$CC" in +*icc*) + BASECFLAGS="$BASECFLAGS -fp-model strict" + ;; +esac + if test "$Py_DEBUG" = 'true'; then : else @@ -1394,6 +1417,12 @@ ;; esac ;; + *icc*) + PGO_PROF_GEN_FLAG="-prof-gen" + PGO_PROF_USE_FLAG="-prof-use" + LLVM_PROF_MERGER="true" + LLVM_PROF_FILE="" + ;; esac -- Repository URL: https://hg.python.org/cpython

1 0

Benchmark Results for Python 2.7 2015-12-21
by lp_benchmark_robot＠intel.com 21 Dec '15

21 Dec '15

Results for project Python 2.7, build date 2015-12-21 04:00:51 +0000 commit: b87fe3e4a3d87e855e8e640da0d9a99ec3b7bd95 revision date: 2015-12-20 09:40:00 +0000 environment: Haswell-EP cpu: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB mem: 128 GB os: CentOS 7.1 kernel: Linux 3.10.0-229.4.2.el7.x86_64 Baseline results were generated using release v2.7.10, with hash 15c95b7d81dcf821daade360741e00714667653f from 2015-05-23 16:02:14+00:00 ---------------------------------------------------------------------------------- benchmark relative change since change since current rev run std_dev* last run baseline with PGO ---------------------------------------------------------------------------------- :-) django_v2 0.43% 2.35% 4.33% 3.53% :-) pybench 0.14% 0.24% 6.25% 4.95% :-( regex_v8 1.06% 0.39% -2.14% 10.50% :-) nbody 0.32% -0.37% 7.27% 2.52% :-) json_dump_v2 0.26% 3.81% 5.40% 8.03% :-( normal_startup 1.75% -0.09% -5.39% 2.30% :-| ssbench 0.19% -0.09% 1.36% 0.83% ---------------------------------------------------------------------------------- Note: Benchmark results for ssbench are measured in requests/second while all other are measured in seconds. * Relative Standard Deviation (Standard Deviation/Average) Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

Benchmark Results for Python Default 2015-12-21
by lp_benchmark_robot＠intel.com 21 Dec '15

21 Dec '15

Results for project Python default, build date 2015-12-21 03:09:06 +0000 commit: af0b95fb1c195b0b24816df0017666b1b59e7bc3 revision date: 2015-12-20 14:37:21 +0000 environment: Haswell-EP cpu: Intel(R) Xeon(R) CPU E5-2699 v3 @ 2.30GHz 2x18 cores, stepping 2, LLC 45 MB mem: 128 GB os: CentOS 7.1 kernel: Linux 3.10.0-229.4.2.el7.x86_64 Baseline results were generated using release v3.4.3, with hash b4cbecbc0781e89a309d03b60a1f75f8499250e6 from 2015-02-25 12:15:33+00:00 ---------------------------------------------------------------------------------- benchmark relative change since change since current rev run std_dev* last run baseline with PGO ---------------------------------------------------------------------------------- :-) django_v2 0.48% -1.93% 9.50% 16.12% :-| pybench 0.11% 0.23% -1.37% 6.20% :-( regex_v8 2.97% -0.05% -4.18% 3.43% :-( nbody 0.24% -2.35% -1.09% 8.66% :-| json_dump_v2 0.27% 0.58% -1.14% 10.30% :-| normal_startup 0.98% -0.00% -1.36% 4.66% ---------------------------------------------------------------------------------- Note: Benchmark results are measured in seconds. * Relative Standard Deviation (Standard Deviation/Average) Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

cpython: Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster.
by serhiy.storchaka 21 Dec '15

21 Dec '15

https://hg.python.org/cpython/rev/090c3e3a648d changeset: 99656:090c3e3a648d user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Dec 21 12:57:27 2015 +0200 summary: Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster. files: Misc/NEWS | 2 + Modules/_elementtree.c | 78 ++++++++++++++++++++--------- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,8 @@ Library ------- +- Issue #25869: Optimized deepcopying ElementTree; it is now 20 times faster. + - Issue #25873: Optimized iterating ElementTree. Iterating elements Element.iter() is now 40% faster, iterating text Element.itertext() is now up to 2.5 times faster. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -129,30 +129,6 @@ /* helpers */ LOCAL(PyObject*) -deepcopy(PyObject* object, PyObject* memo) -{ - /* do a deep copy of the given object */ - PyObject* args; - PyObject* result; - elementtreestate *st = ET_STATE_GLOBAL; - - if (!st->deepcopy_obj) { - PyErr_SetString( - PyExc_RuntimeError, - "deepcopy helper not found" - ); - return NULL; - } - - args = PyTuple_Pack(2, object, memo); - if (!args) - return NULL; - result = PyObject_CallObject(st->deepcopy_obj, args); - Py_DECREF(args); - return result; -} - -LOCAL(PyObject*) list_join(PyObject* list) { /* join list elements (destroying the list in the process) */ @@ -748,6 +724,9 @@ return (PyObject*) element; } +/* Helper for a deep copy. */ +LOCAL(PyObject *) deepcopy(PyObject *, PyObject *); + /*[clinic input] _elementtree.Element.__deepcopy__ @@ -838,6 +817,57 @@ return NULL; } +LOCAL(PyObject *) +deepcopy(PyObject *object, PyObject *memo) +{ + /* do a deep copy of the given object */ + PyObject *args; + PyObject *result; + elementtreestate *st; + + /* Fast paths */ + if (object == Py_None || PyUnicode_CheckExact(object)) { + Py_INCREF(object); + return object; + } + + if (Py_REFCNT(object) == 1) { + if (PyDict_CheckExact(object)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + int simple = 1; + while (PyDict_Next(object, &pos, &key, &value)) { + if (!PyUnicode_CheckExact(key) || !PyUnicode_CheckExact(value)) { + simple = 0; + break; + } + } + if (simple) + return PyDict_Copy(object); + /* Fall through to general case */ + } + else if (Element_CheckExact(object)) { + return _elementtree_Element___deepcopy__((ElementObject *)object, memo); + } + } + + /* General case */ + st = ET_STATE_GLOBAL; + if (!st->deepcopy_obj) { + PyErr_SetString(PyExc_RuntimeError, + "deepcopy helper not found"); + return NULL; + } + + args = PyTuple_Pack(2, object, memo); + if (!args) + return NULL; + result = PyObject_CallObject(st->deepcopy_obj, args); + Py_DECREF(args); + return result; +} + + /*[clinic input] _elementtree.Element.__sizeof__ -> Py_ssize_t -- Repository URL: https://hg.python.org/cpython

1 0

cpython: Issue #25873: Optimized iterating ElementTree.
by serhiy.storchaka 21 Dec '15

21 Dec '15

https://hg.python.org/cpython/rev/5a5d5268afd5 changeset: 99655:5a5d5268afd5 user: Serhiy Storchaka <storchaka(a)gmail.com> date: Mon Dec 21 12:43:54 2015 +0200 summary: Issue #25873: Optimized iterating ElementTree. Iterating elements Element.iter() is now 40% faster, iterating text Element.itertext() is now up to 2.5 times faster. files: Misc/NEWS | 4 + Modules/_elementtree.c | 261 +++++++++++----------------- 2 files changed, 106 insertions(+), 159 deletions(-) diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -115,6 +115,10 @@ Library ------- +- Issue #25873: Optimized iterating ElementTree. Iterating elements + Element.iter() is now 40% faster, iterating text Element.itertext() + is now up to 2.5 times faster. + - Issue #25902: Fixed various refcount issues in ElementTree iteration. - Issue #22227: The TarFile iterator is reimplemented using generator. diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -1986,22 +1986,22 @@ * pre-order traversal. To keep track of which sub-element should be returned * next, a stack of parents is maintained. This is a standard stack-based * iterative pre-order traversal of a tree. - * The stack is managed using a single-linked list starting at parent_stack. - * Each stack node contains the saved parent to which we should return after + * The stack is managed using a continuous array. + * Each stack item contains the saved parent to which we should return after * the current one is exhausted, and the next child to examine in that parent. */ typedef struct ParentLocator_t { ElementObject *parent; Py_ssize_t child_index; - struct ParentLocator_t *next; } ParentLocator; typedef struct { PyObject_HEAD ParentLocator *parent_stack; + Py_ssize_t parent_stack_used; + Py_ssize_t parent_stack_size; ElementObject *root_element; PyObject *sought_tag; - int root_done; int gettext; } ElementIterObject; @@ -2009,13 +2009,11 @@ static void elementiter_dealloc(ElementIterObject *it) { - ParentLocator *p = it->parent_stack; - while (p) { - ParentLocator *temp = p; - Py_XDECREF(p->parent); - p = p->next; - PyObject_Free(temp); - } + Py_ssize_t i = it->parent_stack_used; + it->parent_stack_used = 0; + while (i--) + Py_XDECREF(it->parent_stack[i].parent); + PyMem_Free(it->parent_stack); Py_XDECREF(it->sought_tag); Py_XDECREF(it->root_element); @@ -2027,11 +2025,9 @@ static int elementiter_traverse(ElementIterObject *it, visitproc visit, void *arg) { - ParentLocator *p = it->parent_stack; - while (p) { - Py_VISIT(p->parent); - p = p->next; - } + Py_ssize_t i = it->parent_stack_used; + while (i--) + Py_VISIT(it->parent_stack[i].parent); Py_VISIT(it->root_element); Py_VISIT(it->sought_tag); @@ -2040,17 +2036,25 @@ /* Helper function for elementiter_next. Add a new parent to the parent stack. */ -static ParentLocator * -parent_stack_push_new(ParentLocator *stack, ElementObject *parent) +static int +parent_stack_push_new(ElementIterObject *it, ElementObject *parent) { - ParentLocator *new_node = PyObject_Malloc(sizeof(ParentLocator)); - if (new_node) { - new_node->parent = parent; - Py_INCREF(parent); - new_node->child_index = 0; - new_node->next = stack; + ParentLocator *item; + + if (it->parent_stack_used >= it->parent_stack_size) { + Py_ssize_t new_size = it->parent_stack_size * 2; /* never overflow */ + ParentLocator *parent_stack = it->parent_stack; + PyMem_Resize(parent_stack, ParentLocator, new_size); + if (parent_stack == NULL) + return -1; + it->parent_stack = parent_stack; + it->parent_stack_size = new_size; } - return new_node; + item = it->parent_stack + it->parent_stack_used++; + Py_INCREF(parent); + item->parent = parent; + item->child_index = 0; + return 0; } static PyObject * @@ -2067,151 +2071,91 @@ * - itertext() also has to handle tail, after finishing with all the * children of a node. */ - ElementObject *cur_parent; - Py_ssize_t child_index; int rc; ElementObject *elem; + PyObject *text; while (1) { /* Handle the case reached in the beginning and end of iteration, where - * the parent stack is empty. The root_done flag gives us indication - * whether we've just started iterating (so root_done is 0), in which - * case the root is returned. If root_done is 1 and we're here, the + * the parent stack is empty. If root_element is NULL and we're here, the * iterator is exhausted. */ - if (!it->parent_stack->parent) { - if (it->root_done) { + if (!it->parent_stack_used) { + if (!it->root_element) { PyErr_SetNone(PyExc_StopIteration); return NULL; - } else { - elem = it->root_element; - it->parent_stack = parent_stack_push_new(it->parent_stack, - elem); - if (!it->parent_stack) { - PyErr_NoMemory(); - return NULL; - } - - Py_INCREF(elem); - it->root_done = 1; - rc = (it->sought_tag == Py_None); - if (!rc) { - rc = PyObject_RichCompareBool(elem->tag, - it->sought_tag, Py_EQ); - if (rc < 0) { - Py_DECREF(elem); - return NULL; - } - } - if (rc) { - if (it->gettext) { - PyObject *text = element_get_text(elem); - if (!text) { - Py_DECREF(elem); - return NULL; - } - Py_INCREF(text); - Py_DECREF(elem); - rc = PyObject_IsTrue(text); - if (rc > 0) - return text; - Py_DECREF(text); - if (rc < 0) - return NULL; - } else { - return (PyObject *)elem; - } - } - else { - Py_DECREF(elem); - } } + + elem = it->root_element; /* steals a reference */ + it->root_element = NULL; } - - /* See if there are children left to traverse in the current parent. If - * yes, visit the next child. If not, pop the stack and try again. - */ - cur_parent = it->parent_stack->parent; - child_index = it->parent_stack->child_index; - if (cur_parent->extra && child_index < cur_parent->extra->length) { - elem = (ElementObject *)cur_parent->extra->children[child_index]; - it->parent_stack->child_index++; - it->parent_stack = parent_stack_push_new(it->parent_stack, - elem); - if (!it->parent_stack) { - PyErr_NoMemory(); - return NULL; - } - - Py_INCREF(elem); - if (it->gettext) { - PyObject *text = element_get_text(elem); - if (!text) { - Py_DECREF(elem); - return NULL; - } - Py_INCREF(text); - Py_DECREF(elem); - rc = PyObject_IsTrue(text); - if (rc > 0) - return text; - Py_DECREF(text); - if (rc < 0) - return NULL; - } else { - rc = (it->sought_tag == Py_None); - if (!rc) { - rc = PyObject_RichCompareBool(elem->tag, - it->sought_tag, Py_EQ); - if (rc < 0) { - Py_DECREF(elem); - return NULL; - } - } - if (rc) { - return (PyObject *)elem; + else { + /* See if there are children left to traverse in the current parent. If + * yes, visit the next child. If not, pop the stack and try again. + */ + ParentLocator *item = &it->parent_stack[it->parent_stack_used - 1]; + Py_ssize_t child_index = item->child_index; + ElementObjectExtra *extra; + elem = item->parent; + extra = elem->extra; + if (!extra || child_index >= extra->length) { + it->parent_stack_used--; + /* Note that extra condition on it->parent_stack_used here; + * this is because itertext() is supposed to only return *inner* + * text, not text following the element it began iteration with. + */ + if (it->gettext && it->parent_stack_used) { + text = element_get_tail(elem); + goto gettext; } Py_DECREF(elem); + continue; } + + elem = (ElementObject *)extra->children[child_index]; + item->child_index++; + Py_INCREF(elem); + } + + if (parent_stack_push_new(it, elem) < 0) { + Py_DECREF(elem); + PyErr_NoMemory(); + return NULL; + } + if (it->gettext) { + text = element_get_text(elem); + goto gettext; + } + + if (it->sought_tag == Py_None) + return (PyObject *)elem; + + rc = PyObject_RichCompareBool(elem->tag, it->sought_tag, Py_EQ); + if (rc > 0) + return (PyObject *)elem; + + Py_DECREF(elem); + if (rc < 0) + return NULL; + continue; + +gettext: + if (!text) { + Py_DECREF(elem); + return NULL; + } + if (text == Py_None) { + Py_DECREF(elem); } else { - PyObject *tail; - ParentLocator *next; - if (it->gettext) { - Py_INCREF(cur_parent); - tail = element_get_tail(cur_parent); - if (!tail) { - Py_DECREF(cur_parent); - return NULL; - } - Py_INCREF(tail); - Py_DECREF(cur_parent); - } - else { - tail = Py_None; - Py_INCREF(tail); - } - next = it->parent_stack->next; - cur_parent = it->parent_stack->parent; - PyObject_Free(it->parent_stack); - it->parent_stack = next; - Py_XDECREF(cur_parent); - - /* Note that extra condition on it->parent_stack->parent here; - * this is because itertext() is supposed to only return *inner* - * text, not text following the element it began iteration with. - */ - if (it->parent_stack->parent) { - rc = PyObject_IsTrue(tail); - if (rc > 0) - return tail; - Py_DECREF(tail); - if (rc < 0) - return NULL; - } - else { - Py_DECREF(tail); - } + Py_INCREF(text); + Py_DECREF(elem); + rc = PyObject_IsTrue(text); + if (rc > 0) + return text; + Py_DECREF(text); + if (rc < 0) + return NULL; } } @@ -2263,6 +2207,7 @@ 0, /* tp_new */ }; +#define INIT_PARENT_STACK_SIZE 8 static PyObject * create_elementiter(ElementObject *self, PyObject *tag, int gettext) @@ -2275,22 +2220,20 @@ Py_INCREF(tag); it->sought_tag = tag; - it->root_done = 0; it->gettext = gettext; Py_INCREF(self); it->root_element = self; PyObject_GC_Track(it); - it->parent_stack = PyObject_Malloc(sizeof(ParentLocator)); + it->parent_stack = PyMem_New(ParentLocator, INIT_PARENT_STACK_SIZE); if (it->parent_stack == NULL) { Py_DECREF(it); PyErr_NoMemory(); return NULL; } - it->parent_stack->parent = NULL; - it->parent_stack->child_index = 0; - it->parent_stack->next = NULL; + it->parent_stack_used = 0; + it->parent_stack_size = INIT_PARENT_STACK_SIZE; return (PyObject *)it; } -- Repository URL: https://hg.python.org/cpython

1 0