Mailman 3 August 2017 - Python-checkins

Update the language selection in the docs language switch. (GH-3114)
by Mariatta 17 Aug '17

17 Aug '17

https://github.com/python/cpython/commit/b616b972999cdd5fe54ef8a43c131a27ca… commit: b616b972999cdd5fe54ef8a43c131a27ca538dee branch: master author: Julien Palard <julien(a)palard.fr> committer: Mariatta <Mariatta(a)users.noreply.github.com> date: 2017-08-17T07:24:02-07:00 summary: Update the language selection in the docs language switch. (GH-3114) Change the option for `Français` into `French` to be consistent with the other language selections that are already in English. files: M Doc/tools/static/switchers.js diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index ff23202a5e0..bd31faca4c6 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -20,7 +20,7 @@ var all_languages = { 'en': 'English', - 'fr': 'Français', + 'fr': 'French', 'ja': 'Japanese', };

1 0

bpo-24700: Add a fast path for comparing array.array of equal type (#3009)
by Antoine Pitrou 17 Aug '17

17 Aug '17

https://github.com/python/cpython/commit/7c17e2304b9387f321c813516bf134e4f0… commit: 7c17e2304b9387f321c813516bf134e4f0bd332a branch: master author: Adrian Wielgosik <adrian17(a)users.noreply.github.com> committer: Antoine Pitrou <pitrou(a)free.fr> date: 2017-08-17T14:46:06+02:00 summary: bpo-24700: Add a fast path for comparing array.array of equal type (#3009) files: A Misc/NEWS.d/next/Library/2017-08-08-15-14-34.bpo-24700.44mvNV.rst M Lib/test/test_array.py M Modules/arraymodule.c diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index d67f9195eba..e9218f3dd68 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1342,6 +1342,16 @@ class FPTest(NumberTest): def assertEntryEqual(self, entry1, entry2): self.assertAlmostEqual(entry1, entry2) + def test_nan(self): + a = array.array(self.typecode, [float('nan')]) + b = array.array(self.typecode, [float('nan')]) + self.assertIs(a != b, True) + self.assertIs(a == b, False) + self.assertIs(a > b, False) + self.assertIs(a >= b, False) + self.assertIs(a < b, False) + self.assertIs(a <= b, False) + def test_byteswap(self): a = array.array(self.typecode, self.example) self.assertRaises(TypeError, a.byteswap, 42) diff --git a/Misc/NEWS.d/next/Library/2017-08-08-15-14-34.bpo-24700.44mvNV.rst b/Misc/NEWS.d/next/Library/2017-08-08-15-14-34.bpo-24700.44mvNV.rst new file mode 100644 index 00000000000..a78172e0423 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-08-08-15-14-34.bpo-24700.44mvNV.rst @@ -0,0 +1,2 @@ +Optimize array.array comparison. It is now from 10x up to 70x faster when +comparing arrays holding values of the same integer type. \ No newline at end of file diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 06461c38036..1d9a4f11d87 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -31,6 +31,7 @@ struct arraydescr { int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); + int (*compareitems)(const void *, const void *, Py_ssize_t); const char *formats; int is_integer_type; int is_signed; @@ -518,6 +519,28 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return 0; } +#define DEFINE_COMPAREITEMS(code, type) \ + static int \ + code##_compareitems(const void *lhs, const void *rhs, Py_ssize_t length) \ + { \ + const type *a = lhs, *b = rhs; \ + for (Py_ssize_t i = 0; i < length; ++i) \ + if (a[i] != b[i]) \ + return a[i] < b[i] ? -1 : 1; \ + return 0; \ + } + +DEFINE_COMPAREITEMS(b, signed char) +DEFINE_COMPAREITEMS(BB, unsigned char) +DEFINE_COMPAREITEMS(u, Py_UNICODE) +DEFINE_COMPAREITEMS(h, short) +DEFINE_COMPAREITEMS(HH, unsigned short) +DEFINE_COMPAREITEMS(i, int) +DEFINE_COMPAREITEMS(II, unsigned int) +DEFINE_COMPAREITEMS(l, long) +DEFINE_COMPAREITEMS(LL, unsigned long) +DEFINE_COMPAREITEMS(q, long long) +DEFINE_COMPAREITEMS(QQ, unsigned long long) /* Description of types. * @@ -525,19 +548,19 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) * typecode. */ static const struct arraydescr descriptors[] = { - {'b', 1, b_getitem, b_setitem, "b", 1, 1}, - {'B', 1, BB_getitem, BB_setitem, "B", 1, 0}, - {'u', sizeof(Py_UNICODE), u_getitem, u_setitem, "u", 0, 0}, - {'h', sizeof(short), h_getitem, h_setitem, "h", 1, 1}, - {'H', sizeof(short), HH_getitem, HH_setitem, "H", 1, 0}, - {'i', sizeof(int), i_getitem, i_setitem, "i", 1, 1}, - {'I', sizeof(int), II_getitem, II_setitem, "I", 1, 0}, - {'l', sizeof(long), l_getitem, l_setitem, "l", 1, 1}, - {'L', sizeof(long), LL_getitem, LL_setitem, "L", 1, 0}, - {'q', sizeof(long long), q_getitem, q_setitem, "q", 1, 1}, - {'Q', sizeof(long long), QQ_getitem, QQ_setitem, "Q", 1, 0}, - {'f', sizeof(float), f_getitem, f_setitem, "f", 0, 0}, - {'d', sizeof(double), d_getitem, d_setitem, "d", 0, 0}, + {'b', 1, b_getitem, b_setitem, b_compareitems, "b", 1, 1}, + {'B', 1, BB_getitem, BB_setitem, BB_compareitems, "B", 1, 0}, + {'u', sizeof(Py_UNICODE), u_getitem, u_setitem, u_compareitems, "u", 0, 0}, + {'h', sizeof(short), h_getitem, h_setitem, h_compareitems, "h", 1, 1}, + {'H', sizeof(short), HH_getitem, HH_setitem, HH_compareitems, "H", 1, 0}, + {'i', sizeof(int), i_getitem, i_setitem, i_compareitems, "i", 1, 1}, + {'I', sizeof(int), II_getitem, II_setitem, II_compareitems, "I", 1, 0}, + {'l', sizeof(long), l_getitem, l_setitem, l_compareitems, "l", 1, 1}, + {'L', sizeof(long), LL_getitem, LL_setitem, LL_compareitems, "L", 1, 0}, + {'q', sizeof(long long), q_getitem, q_setitem, q_compareitems, "q", 1, 1}, + {'Q', sizeof(long long), QQ_getitem, QQ_setitem, QQ_compareitems, "Q", 1, 0}, + {'f', sizeof(float), f_getitem, f_setitem, NULL, "f", 0, 0}, + {'d', sizeof(double), d_getitem, d_setitem, NULL, "d", 0, 0}, {'\0', 0, 0, 0, 0, 0, 0} /* Sentinel */ }; @@ -664,6 +687,31 @@ array_richcompare(PyObject *v, PyObject *w, int op) return res; } + if (va->ob_descr == wa->ob_descr && va->ob_descr->compareitems != NULL) { + /* Fast path: + arrays with same types can have their buffers compared directly */ + Py_ssize_t common_length = Py_MIN(Py_SIZE(va), Py_SIZE(wa)); + int result = va->ob_descr->compareitems(va->ob_item, wa->ob_item, + common_length); + if (result == 0) + goto compare_sizes; + + int cmp; + switch (op) { + case Py_LT: cmp = result < 0; break; + case Py_LE: cmp = result <= 0; break; + case Py_EQ: cmp = result == 0; break; + case Py_NE: cmp = result != 0; break; + case Py_GT: cmp = result > 0; break; + case Py_GE: cmp = result >= 0; break; + default: return NULL; /* cannot happen */ + } + PyObject *res = cmp ? Py_True : Py_False; + Py_INCREF(res); + return res; + } + + /* Search for the first index where items are different */ k = 1; for (i = 0; i < Py_SIZE(va) && i < Py_SIZE(wa); i++) { @@ -685,14 +733,17 @@ array_richcompare(PyObject *v, PyObject *w, int op) if (k) { /* No more items to compare -- compare sizes */ + compare_sizes: ; Py_ssize_t vs = Py_SIZE(va); Py_ssize_t ws = Py_SIZE(wa); int cmp; switch (op) { case Py_LT: cmp = vs < ws; break; case Py_LE: cmp = vs <= ws; break; - case Py_EQ: cmp = vs == ws; break; - case Py_NE: cmp = vs != ws; break; + /* If the lengths were not equal, + the earlier fast-path check would have caught that. */ + case Py_EQ: assert(vs == ws); cmp = 1; break; + case Py_NE: assert(vs == ws); cmp = 0; break; case Py_GT: cmp = vs > ws; break; case Py_GE: cmp = vs >= ws; break; default: return NULL; /* cannot happen */

1 0

bpo-29619: Do not use HAVE_LARGEFILE_SUPPORT for type conversions (GH-1666) (#3102)
by Victor Stinner 17 Aug '17

17 Aug '17

https://github.com/python/cpython/commit/ffbb6f7334ccf54f09dcc9e760766d8619… commit: ffbb6f7334ccf54f09dcc9e760766d861928f13e branch: 3.6 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-17T14:33:06+02:00 summary: bpo-29619: Do not use HAVE_LARGEFILE_SUPPORT for type conversions (GH-1666) (#3102) Use only the LongLong form for the conversions (cherry picked from commit 50e86033de85294d87b7e942701d456342abde8e) files: M Modules/posixmodule.c diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 97b57198cd4..111e4c9fde5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1934,14 +1934,8 @@ _pystat_fromstructstat(STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); -#if defined(HAVE_LARGEFILE_SUPPORT) || defined(MS_WINDOWS) Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(st->st_ino)); - PyStructSequence_SET_ITEM(v, 1, - PyLong_FromUnsignedLongLong(st->st_ino)); -#else - Py_BUILD_ASSERT(sizeof(unsigned long) >= sizeof(st->st_ino)); - PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLong(st->st_ino)); -#endif + PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLongLong(st->st_ino)); #ifdef MS_WINDOWS PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); #else @@ -1955,12 +1949,8 @@ _pystat_fromstructstat(STRUCT_STAT *st) PyStructSequence_SET_ITEM(v, 4, _PyLong_FromUid(st->st_uid)); PyStructSequence_SET_ITEM(v, 5, _PyLong_FromGid(st->st_gid)); #endif -#ifdef HAVE_LARGEFILE_SUPPORT - PyStructSequence_SET_ITEM(v, 6, - PyLong_FromLongLong((long long)st->st_size)); -#else - PyStructSequence_SET_ITEM(v, 6, PyLong_FromLong(st->st_size)); -#endif + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(st->st_size)); + PyStructSequence_SET_ITEM(v, 6, PyLong_FromLongLong(st->st_size)); #if defined(HAVE_STAT_TV_NSEC) ansec = st->st_atim.tv_nsec; @@ -11484,11 +11474,8 @@ DirEntry_inode(DirEntry *self) Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->win32_file_index)); return PyLong_FromUnsignedLongLong(self->win32_file_index); #else /* POSIX */ -#ifdef HAVE_LARGEFILE_SUPPORT - return PyLong_FromLongLong((long long)self->d_ino); -#else - return PyLong_FromLong((long)self->d_ino); -#endif + Py_BUILD_ASSERT(sizeof(unsigned long long) >= sizeof(self->d_ino)); + return PyLong_FromUnsignedLongLong(self->d_ino); #endif }

1 0

Daily reference leaks (4243df51fe43): sum=7
by solipsis＠pitrou.net 17 Aug '17

17 Aug '17

results for 4243df51fe43 on branch "default" -------------------------------------------- test_asyncio leaked [0, 0, 3] memory blocks, sum=3 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogBRVFBa', '--timeout', '7200']

1 0

[65 flat] Results for Python (master branch) 2017-08-16
by lp_benchmark_robot＠intel.com 17 Aug '17

17 Aug '17

Results for project python/master, build date: 2017-08-16 03:03:58-07:00. - commit: 7cc3399 - previous commit: 7b40cb7 - revision date: 2017-08-16 11:02:05+02:00 - environment: Broadwell-EP - cpu: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz 2x22 cores, stepping 1, LLC 55 MB - mem: 128 GB - os: Ubuntu 16.04.2 LTS - kernel: 4.4.0-62-generic x86_64 GNU/Linux Baseline results were generated using release v3.6.0, with hash 5c4568a from 2016-12-22 23:38:47+00:00. +-----+------------------------+--------+------------+------------+------------+ | | |relative|change since|change since|current rev | | | benchmark|std_dev*| last run | baseline |run with PGO| +-----+------------------------+--------+------------+------------+------------+ | :-| | 2to3| 1.152% | +0.031% | +4.257% | +8.173% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method| 1.709% | -0.198% | +21.573% | +11.282% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_slots| 1.723% | +0.702% | +24.323% | +10.077% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_unknown| 0.672% | -0.188% | +21.028% | +7.930% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_simple| 2.369% | -0.336% | +2.595% | +16.066% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chameleon| 1.984% | -0.321% | +11.034% | +10.838% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chaos| 1.848% | -0.013% | +6.199% | +10.469% | +-----+------------------------+--------+------------+------------+------------+ | :-| | crypto_pyaes| 0.883% | +0.056% | +4.273% | +5.507% | +-----+------------------------+--------+------------+------------+------------+ | :-| | deltablue| 5.342% | -0.647% | +5.793% | +19.776% | +-----+------------------------+--------+------------+------------+------------+ | :-| | django_template| 3.334% | +0.167% | +9.329% | +13.980% | +-----+------------------------+--------+------------+------------+------------+ | :-| | dulwich_log| 1.633% | -0.270% | +4.089% | +6.798% | +-----+------------------------+--------+------------+------------+------------+ | :-| | fannkuch| 1.320% | -0.955% | +4.137% | +4.027% | +-----+------------------------+--------+------------+------------+------------+ | :-| | float| 0.608% | +0.368% | +3.236% | +6.222% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_text| 1.402% | -0.370% | +8.962% | +10.713% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_xml| 1.956% | -0.038% | +7.918% | +7.664% | +-----+------------------------+--------+------------+------------+------------+ | :-| | go| 0.714% | +0.056% | +6.408% | +11.480% | +-----+------------------------+--------+------------+------------+------------+ | :-| | hexiom| 0.438% | -0.054% | +9.060% | +12.070% | +-----+------------------------+--------+------------+------------+------------+ | :-| | html5lib| 3.437% | -0.123% | +7.794% | +11.028% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_dumps| 2.699% | -1.161% | +3.546% | +10.117% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_loads| 2.305% | -0.018% | +2.770% | +11.345% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_format| 1.314% | +0.580% | +8.532% | +12.805% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_silent| 2.537% | -0.009% | +47.219% | +11.156% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_simple| 1.522% | +0.360% | +9.497% | +12.640% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mako| 0.439% | +0.036% | +18.764% | +11.115% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mdp| 6.431% | -0.555% | +5.078% | +13.332% | +-----+------------------------+--------+------------+------------+------------+ | :-| | meteor_contest| 1.563% | -0.580% | +3.384% | +5.400% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nbody| 0.974% | -0.102% | -2.002% | +2.684% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nqueens| 0.758% | +0.574% | +2.707% | +6.271% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pathlib| 1.134% | +0.164% | +5.916% | +9.431% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle| 1.860% | -0.123% | +0.814% | +22.410% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_dict| 0.240% | -0.119% | +2.430% | +19.292% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_list| 0.790% | +0.229% | +5.133% | +18.193% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_pure_python| 2.174% | +0.678% | +12.660% | +9.274% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pidigits| 0.137% | -0.033% | +0.299% | +10.052% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup| 0.135% | -0.053% | +8.878% | +4.983% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup_no_site| 0.096% | -0.084% | +0.903% | +4.776% | +-----+------------------------+--------+------------+------------+------------+ | :-| | raytrace| 1.381% | -0.151% | +9.198% | +13.199% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_compile| 4.873% | +1.523% | -9.251% | +11.600% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_dna| 0.434% | +0.070% | +1.521% | +8.359% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_effbot| 1.573% | -0.385% | +0.743% | +3.360% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_v8| 0.872% | +0.071% | +11.292% | +3.864% | +-----+------------------------+--------+------------+------------+------------+ | :-| | richards| 1.652% | +0.090% | +7.726% | +14.711% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_fft| 0.479% | +0.366% | +0.801% | +2.091% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_lu| 2.007% | -0.714% | +26.778% | +8.282% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_monte_carlo| 1.439% | +0.591% | +3.656% | +6.295% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sor| 0.387% | +0.212% | +13.324% | +9.544% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sparse_mat_mult| 1.762% | +0.244% | -0.389% | -4.261% | +-----+------------------------+--------+------------+------------+------------+ | :-| | spectral_norm| 1.605% | +0.087% | +5.632% | +2.312% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_declarative| 1.284% | -0.194% | +5.197% | +6.764% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_imperative| 3.271% | -0.526% | +5.356% | +3.977% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlite_synth| 4.231% | -1.285% | +2.974% | +6.806% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_expand| 2.300% | -0.571% | +12.252% | +7.873% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_integrate| 1.484% | -0.033% | +10.433% | +6.434% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_str| 3.601% | -0.325% | +12.112% | +8.021% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_sum| 4.284% | +0.012% | +13.680% | +8.941% | +-----+------------------------+--------+------------+------------+------------+ | :-| | telco| 5.570% | -0.904% | +23.094% | +10.781% | +-----+------------------------+--------+------------+------------+------------+ | :-| | tornado_http| 1.057% | +0.275% | +6.094% | +4.711% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpack_sequence| 1.170% | +0.371% | +0.905% | -1.236% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle| 8.489% | +1.815% | +8.001% | +21.395% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_list| 2.026% | +0.366% | -3.410% | +18.766% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_pure_python| 2.652% | +0.106% | +6.854% | +6.123% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_generate| 1.304% | +0.116% | +5.721% | +7.843% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_iterparse| 1.946% | +0.003% | +2.225% | +6.764% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_parse| 1.402% | -0.115% | -5.385% | +11.070% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_process| 1.152% | +0.372% | +6.694% | +8.101% | +-----+------------------------+--------+------------+------------+------------+ * Relative Standard Deviation (Standard Deviation/Average) If this is not displayed properly please visit our results page here: http://languagesperformance.intel.com/65-flat-results-for-python-master-bra… Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

bpo-18966: non-daemonic threads created by a multiprocessing.Process should be joined on exit (#3111)
by Antoine Pitrou 16 Aug '17

16 Aug '17

https://github.com/python/cpython/commit/ee84a608587b930176d37303afae8a4358… commit: ee84a608587b930176d37303afae8a4358e15990 branch: master author: Antoine Pitrou <pitrou(a)free.fr> committer: GitHub <noreply(a)github.com> date: 2017-08-16T20:53:28+02:00 summary: bpo-18966: non-daemonic threads created by a multiprocessing.Process should be joined on exit (#3111) * bpo-18966: non-daemonic threads created by a multiprocessing.Process should be joined on exit * Add NEWS blurb files: A Misc/NEWS.d/next/Library/2017-08-16-20-28-06.bpo-18966.mjHWk2.rst M Lib/multiprocessing/process.py M Lib/test/_test_multiprocessing.py M Lib/threading.py diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 8e500dc93dc..8fff3e105ea 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -17,6 +17,7 @@ import sys import signal import itertools +import threading from _weakrefset import WeakSet # @@ -311,6 +312,7 @@ def _bootstrap(self): sys.stderr.write('Process %s:\n' % self.name) traceback.print_exc() finally: + threading._shutdown() util.info('process exiting with exitcode %d' % exitcode) sys.stdout.flush() sys.stderr.flush() diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index dce62df1d02..d6fe7d62675 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -542,6 +542,32 @@ def test_child_fd_inflation(self): p.join() close_queue(q) + @classmethod + def _test_wait_for_threads(self, evt): + def func1(): + time.sleep(0.5) + evt.set() + + def func2(): + time.sleep(20) + evt.clear() + + threading.Thread(target=func1).start() + threading.Thread(target=func2, daemon=True).start() + + def test_wait_for_threads(self): + # A child process should wait for non-daemonic threads to end + # before exiting + if self.TYPE == 'threads': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + evt = self.Event() + proc = self.Process(target=self._test_wait_for_threads, args=(evt,)) + proc.start() + proc.join() + self.assertTrue(evt.is_set()) + + # # # diff --git a/Lib/threading.py b/Lib/threading.py index 92c2ab365b4..06dbc682834 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1284,6 +1284,9 @@ def _shutdown(): # the main thread's tstate_lock - that won't happen until the interpreter # is nearly dead. So we release it here. Note that just calling _stop() # isn't enough: other threads may already be waiting on _tstate_lock. + if _main_thread._is_stopped: + # _shutdown() was already called + return tlock = _main_thread._tstate_lock # The main thread isn't finished yet, so its thread state lock can't have # been released. diff --git a/Misc/NEWS.d/next/Library/2017-08-16-20-28-06.bpo-18966.mjHWk2.rst b/Misc/NEWS.d/next/Library/2017-08-16-20-28-06.bpo-18966.mjHWk2.rst new file mode 100644 index 00000000000..53bea710127 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-08-16-20-28-06.bpo-18966.mjHWk2.rst @@ -0,0 +1,2 @@ +Non-daemonic threads created by a multiprocessing.Process are now joined on +child exit.

1 0

bpo-31069, test_multiprocessing: Fix dangling process (#3103) (#3105)
by Victor Stinner 16 Aug '17

16 Aug '17

https://github.com/python/cpython/commit/883520a846d87f01a14636978cfa9cee1e… commit: 883520a846d87f01a14636978cfa9cee1ecc07c1 branch: 2.7 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-16T13:14:40+02:00 summary: bpo-31069, test_multiprocessing: Fix dangling process (#3103) (#3105) Fix a warning about dangling processes in test_rapid_restart() of _test_multiprocessing: join the process. (cherry picked from commit 17657bb9458ff8f8804b7637d61686a68f4b9471) files: M Lib/test/test_multiprocessing.py diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 69bd8f79a85..a77bfee6718 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -1583,12 +1583,13 @@ def test_rapid_restart(self): manager.start() p = self.Process(target=self._putter, args=(manager.address, authkey)) - p.daemon = True p.start() + p.join() queue = manager.get_queue() self.assertEqual(queue.get(), 'hello world') del queue manager.shutdown() + manager = QueueManager( address=addr, authkey=authkey, serializer=SERIALIZER) manager.start()

1 0

bpo-31069, test_multiprocessing: Fix dangling process (#3103) (#3104)
by Victor Stinner 16 Aug '17

16 Aug '17

https://github.com/python/cpython/commit/78c66a641a0d4fe61356ebfae8772e4fae… commit: 78c66a641a0d4fe61356ebfae8772e4fae24dcb7 branch: 3.6 author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-16T13:14:38+02:00 summary: bpo-31069, test_multiprocessing: Fix dangling process (#3103) (#3104) Fix a warning about dangling processes in test_rapid_restart() of _test_multiprocessing: join the process. (cherry picked from commit 17657bb9458ff8f8804b7637d61686a68f4b9471) files: M Lib/test/_test_multiprocessing.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4d3c6558374..db30e6b9d22 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2425,12 +2425,13 @@ def test_rapid_restart(self): manager.start() p = self.Process(target=self._putter, args=(manager.address, authkey)) - p.daemon = True p.start() + p.join() queue = manager.get_queue() self.assertEqual(queue.get(), 'hello world') del queue manager.shutdown() + manager = QueueManager( address=addr, authkey=authkey, serializer=SERIALIZER) try:

1 0

bpo-31069, test_multiprocessing: Fix dangling process (#3103)
by Victor Stinner 16 Aug '17

16 Aug '17

https://github.com/python/cpython/commit/17657bb9458ff8f8804b7637d61686a68f… commit: 17657bb9458ff8f8804b7637d61686a68f4b9471 branch: master author: Victor Stinner <victor.stinner(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-16T12:46:04+02:00 summary: bpo-31069, test_multiprocessing: Fix dangling process (#3103) Fix a warning about dangling processes in test_rapid_restart() of _test_multiprocessing: join the process. files: M Lib/test/_test_multiprocessing.py diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index cce3c0287e9..dce62df1d02 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2596,12 +2596,13 @@ def test_rapid_restart(self): manager.start() p = self.Process(target=self._putter, args=(manager.address, authkey)) - p.daemon = True p.start() + p.join() queue = manager.get_queue() self.assertEqual(queue.get(), 'hello world') del queue manager.shutdown() + manager = QueueManager( address=addr, authkey=authkey, serializer=SERIALIZER) try:

1 0

Daily reference leaks (4243df51fe43): sum=4
by solipsis＠pitrou.net 16 Aug '17

16 Aug '17

results for 4243df51fe43 on branch "default" -------------------------------------------- test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, -2, 0] memory blocks, sum=0 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogMeWcy_', '--timeout', '7200']

1 0