Mailman 3 August 2017 - Python-checkins

Daily reference leaks (4243df51fe43): sum=11
by solipsis＠pitrou.net 30 Aug '17

30 Aug '17

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [0, 0, 7] memory blocks, sum=7 test_functools leaked [0, 3, 1] memory blocks, sum=4 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogdxNLxz', '--timeout', '7200']

1 0

bpo-30581: Windows: os.cpu_count() returns wrong number of processors (#2934)
by Antoine Pitrou 30 Aug '17

30 Aug '17

https://github.com/python/cpython/commit/c67bae04780f9d7590f9f91b4ee5f31c5d… commit: c67bae04780f9d7590f9f91b4ee5f31c5d75b3c3 branch: master author: Christopher Wilcox <git(a)crwilcox.com> committer: Antoine Pitrou <pitrou(a)free.fr> date: 2017-08-30T11:01:08+02:00 summary: bpo-30581: Windows: os.cpu_count() returns wrong number of processors (#2934) * Fixes #30581 by adding a path to use newer GetMaximumProcessorCount API on Windows calls to os.cpu_count() * Add NEWS.d entry for bpo-30581, os.cpu_count on Windows. * Tweak NEWS entry files: A Misc/NEWS.d/next/Windows/2017-08-04-10-05-19.bpo-30581.OQhR7l.rst M Modules/posixmodule.c diff --git a/Misc/NEWS.d/next/Windows/2017-08-04-10-05-19.bpo-30581.OQhR7l.rst b/Misc/NEWS.d/next/Windows/2017-08-04-10-05-19.bpo-30581.OQhR7l.rst new file mode 100644 index 00000000000..6591fa0df00 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2017-08-04-10-05-19.bpo-30581.OQhR7l.rst @@ -0,0 +1,2 @@ +os.cpu_count() now returns the correct number of processors on Windows +when the number of logical processors is greater than 64. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f0577874334..e8138d5d3c3 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11174,9 +11174,22 @@ os_cpu_count_impl(PyObject *module) { int ncpu = 0; #ifdef MS_WINDOWS - SYSTEM_INFO sysinfo; - GetSystemInfo(&sysinfo); - ncpu = sysinfo.dwNumberOfProcessors; + /* Vista is supported and the GetMaximumProcessorCount API is Win7+ + Need to fallback to Vista behavior if this call isn't present */ + HINSTANCE hKernel32; + hKernel32 = GetModuleHandleW(L"KERNEL32"); + + static DWORD(CALLBACK *_GetMaximumProcessorCount)(WORD) = NULL; + *(FARPROC*)&_GetMaximumProcessorCount = GetProcAddress(hKernel32, + "GetMaximumProcessorCount"); + if (_GetMaximumProcessorCount != NULL) { + ncpu = _GetMaximumProcessorCount(ALL_PROCESSOR_GROUPS); + } + else { + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + ncpu = sysinfo.dwNumberOfProcessors; + } #elif defined(__hpux) ncpu = mpctl(MPC_GETNUMSPUS, NULL, NULL); #elif defined(HAVE_SYSCONF) && defined(_SC_NPROCESSORS_ONLN)

1 0

[3.6] bpo-31051: Rearrange IDLE condigdialog GenPage into Window, Editor, and Help sections. (GH-3239) (#3240)
by Terry Jan Reedy 30 Aug '17

30 Aug '17

https://github.com/python/cpython/commit/87c50245b1ba21469cc2e4e84cd5e5cd54… commit: 87c50245b1ba21469cc2e4e84cd5e5cd54ff954d branch: 3.6 author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2017-08-30T02:47:24-04:00 summary: [3.6] bpo-31051: Rearrange IDLE condigdialog GenPage into Window, Editor, and Help sections. (GH-3239) (#3240) (cherry picked from commit 390ead) files: A Misc/NEWS.d/next/IDLE/2017-08-30-00-06-58.bpo-31051.50Jp_Q.rst M Lib/idlelib/configdialog.py diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 8afc9e6d692..4ef6197e714 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -1739,20 +1739,22 @@ def create_page_general(self): rewrite changes['main']['HelpFiles']. Widgets for GenPage(Frame): (*) widgets bound to self - frame_run: LabelFrame - startup_title: Label - (*)startup_editor_on: Radiobutton - startup_edit - (*)startup_shell_on: Radiobutton - startup_edit - frame_save: LabelFrame - run_save_title: Label - (*)save_ask_on: Radiobutton - autosave - (*)save_auto_on: Radiobutton - autosave - frame_win_size: LabelFrame - win_size_title: Label - win_width_title: Label - (*)win_width_int: Entry - win_width - win_height_title: Label - (*)win_height_int: Entry - win_height + frame_window: LabelFrame + frame_run: Frame + startup_title: Label + (*)startup_editor_on: Radiobutton - startup_edit + (*)startup_shell_on: Radiobutton - startup_edit + frame_win_size: Frame + win_size_title: Label + win_width_title: Label + (*)win_width_int: Entry - win_width + win_height_title: Label + (*)win_height_int: Entry - win_height + frame_editor: LabelFrame + frame_save: Frame + run_save_title: Label + (*)save_ask_on: Radiobutton - autosave + (*)save_auto_on: Radiobutton - autosave frame_help: LabelFrame frame_helplist: Frame frame_helplist_buttons: Frame @@ -1771,16 +1773,15 @@ def create_page_general(self): self.win_height = tracers.add( StringVar(self), ('main', 'EditorWindow', 'height')) - # Create widgets: # Section frames. - frame_run = LabelFrame(self, borderwidth=2, relief=GROOVE, - text=' Startup Preferences ') - frame_save = LabelFrame(self, borderwidth=2, relief=GROOVE, - text=' autosave Preferences ') - frame_win_size = Frame(self, borderwidth=2, relief=GROOVE) + frame_window = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Window Preferences') + frame_editor = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Editor Preferences') frame_help = LabelFrame(self, borderwidth=2, relief=GROOVE, text=' Additional Help Sources ') - # frame_run. + # Frame_window. + frame_run = Frame(frame_window, borderwidth=0) startup_title = Label(frame_run, text='At Startup') self.startup_editor_on = Radiobutton( frame_run, variable=self.startup_edit, value=1, @@ -1788,15 +1789,8 @@ def create_page_general(self): self.startup_shell_on = Radiobutton( frame_run, variable=self.startup_edit, value=0, text='Open Shell Window') - # frame_save. - run_save_title = Label(frame_save, text='At Start of Run (F5) ') - self.save_ask_on = Radiobutton( - frame_save, variable=self.autosave, value=0, - text="Prompt to Save") - self.save_auto_on = Radiobutton( - frame_save, variable=self.autosave, value=1, - text='No Prompt') - # frame_win_size. + + frame_win_size = Frame(frame_window, borderwidth=0,) win_size_title = Label( frame_win_size, text='Initial Window Size (in characters)') win_width_title = Label(frame_win_size, text='Width') @@ -1805,6 +1799,17 @@ def create_page_general(self): win_height_title = Label(frame_win_size, text='Height') self.win_height_int = Entry( frame_win_size, textvariable=self.win_height, width=3) + + # Frame_editor. + frame_save = Frame(frame_editor, borderwidth=0) + run_save_title = Label(frame_save, text='At Start of Run (F5) ') + self.save_ask_on = Radiobutton( + frame_save, variable=self.autosave, value=0, + text="Prompt to Save") + self.save_auto_on = Radiobutton( + frame_save, variable=self.autosave, value=1, + text='No Prompt') + # frame_help. frame_helplist = Frame(frame_help) frame_helplist_buttons = Frame(frame_helplist) @@ -1826,25 +1831,27 @@ def create_page_general(self): width=8, command=self.helplist_item_remove) # Pack widgets: - # body. - frame_run.pack(side=TOP, padx=5, pady=5, fill=X) - frame_save.pack(side=TOP, padx=5, pady=5, fill=X) - frame_win_size.pack(side=TOP, padx=5, pady=5, fill=X) + # Body. + frame_window.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_editor.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) # frame_run. + frame_run.pack(side=TOP, padx=5, pady=0, fill=X) startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) self.startup_shell_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) self.startup_editor_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - # frame_save. - run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) # frame_win_size. + frame_win_size.pack(side=TOP, padx=5, pady=0, fill=X) win_size_title.pack(side=LEFT, anchor=W, padx=5, pady=5) self.win_height_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) win_height_title.pack(side=RIGHT, anchor=E, pady=5) self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) win_width_title.pack(side=RIGHT, anchor=E, pady=5) + # frame_save. + frame_save.pack(side=TOP, padx=5, pady=0, fill=X) + run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) # frame_help. frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) diff --git a/Misc/NEWS.d/next/IDLE/2017-08-30-00-06-58.bpo-31051.50Jp_Q.rst b/Misc/NEWS.d/next/IDLE/2017-08-30-00-06-58.bpo-31051.50Jp_Q.rst new file mode 100644 index 00000000000..fe515911b0d --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-08-30-00-06-58.bpo-31051.50Jp_Q.rst @@ -0,0 +1 @@ +Rearrange IDLE condigdialog GenPage into Window, Editor, and Help sections.

1 0

bpo-31051: Rearrange IDLE condigdialog GenPage into Window, Editor, and Help sections. (#3239)
by Terry Jan Reedy 30 Aug '17

30 Aug '17

https://github.com/python/cpython/commit/390eadd6d041611511dc761bc7d4581530… commit: 390eadd6d041611511dc761bc7d4581530dbd287 branch: master author: Terry Jan Reedy <tjreedy(a)udel.edu> committer: GitHub <noreply(a)github.com> date: 2017-08-30T00:59:11-04:00 summary: bpo-31051: Rearrange IDLE condigdialog GenPage into Window, Editor, and Help sections. (#3239) files: A Misc/NEWS.d/next/IDLE/2017-08-30-00-06-58.bpo-31051.50Jp_Q.rst M Lib/idlelib/configdialog.py diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 8afc9e6d692..4ef6197e714 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -1739,20 +1739,22 @@ def create_page_general(self): rewrite changes['main']['HelpFiles']. Widgets for GenPage(Frame): (*) widgets bound to self - frame_run: LabelFrame - startup_title: Label - (*)startup_editor_on: Radiobutton - startup_edit - (*)startup_shell_on: Radiobutton - startup_edit - frame_save: LabelFrame - run_save_title: Label - (*)save_ask_on: Radiobutton - autosave - (*)save_auto_on: Radiobutton - autosave - frame_win_size: LabelFrame - win_size_title: Label - win_width_title: Label - (*)win_width_int: Entry - win_width - win_height_title: Label - (*)win_height_int: Entry - win_height + frame_window: LabelFrame + frame_run: Frame + startup_title: Label + (*)startup_editor_on: Radiobutton - startup_edit + (*)startup_shell_on: Radiobutton - startup_edit + frame_win_size: Frame + win_size_title: Label + win_width_title: Label + (*)win_width_int: Entry - win_width + win_height_title: Label + (*)win_height_int: Entry - win_height + frame_editor: LabelFrame + frame_save: Frame + run_save_title: Label + (*)save_ask_on: Radiobutton - autosave + (*)save_auto_on: Radiobutton - autosave frame_help: LabelFrame frame_helplist: Frame frame_helplist_buttons: Frame @@ -1771,16 +1773,15 @@ def create_page_general(self): self.win_height = tracers.add( StringVar(self), ('main', 'EditorWindow', 'height')) - # Create widgets: # Section frames. - frame_run = LabelFrame(self, borderwidth=2, relief=GROOVE, - text=' Startup Preferences ') - frame_save = LabelFrame(self, borderwidth=2, relief=GROOVE, - text=' autosave Preferences ') - frame_win_size = Frame(self, borderwidth=2, relief=GROOVE) + frame_window = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Window Preferences') + frame_editor = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Editor Preferences') frame_help = LabelFrame(self, borderwidth=2, relief=GROOVE, text=' Additional Help Sources ') - # frame_run. + # Frame_window. + frame_run = Frame(frame_window, borderwidth=0) startup_title = Label(frame_run, text='At Startup') self.startup_editor_on = Radiobutton( frame_run, variable=self.startup_edit, value=1, @@ -1788,15 +1789,8 @@ def create_page_general(self): self.startup_shell_on = Radiobutton( frame_run, variable=self.startup_edit, value=0, text='Open Shell Window') - # frame_save. - run_save_title = Label(frame_save, text='At Start of Run (F5) ') - self.save_ask_on = Radiobutton( - frame_save, variable=self.autosave, value=0, - text="Prompt to Save") - self.save_auto_on = Radiobutton( - frame_save, variable=self.autosave, value=1, - text='No Prompt') - # frame_win_size. + + frame_win_size = Frame(frame_window, borderwidth=0,) win_size_title = Label( frame_win_size, text='Initial Window Size (in characters)') win_width_title = Label(frame_win_size, text='Width') @@ -1805,6 +1799,17 @@ def create_page_general(self): win_height_title = Label(frame_win_size, text='Height') self.win_height_int = Entry( frame_win_size, textvariable=self.win_height, width=3) + + # Frame_editor. + frame_save = Frame(frame_editor, borderwidth=0) + run_save_title = Label(frame_save, text='At Start of Run (F5) ') + self.save_ask_on = Radiobutton( + frame_save, variable=self.autosave, value=0, + text="Prompt to Save") + self.save_auto_on = Radiobutton( + frame_save, variable=self.autosave, value=1, + text='No Prompt') + # frame_help. frame_helplist = Frame(frame_help) frame_helplist_buttons = Frame(frame_helplist) @@ -1826,25 +1831,27 @@ def create_page_general(self): width=8, command=self.helplist_item_remove) # Pack widgets: - # body. - frame_run.pack(side=TOP, padx=5, pady=5, fill=X) - frame_save.pack(side=TOP, padx=5, pady=5, fill=X) - frame_win_size.pack(side=TOP, padx=5, pady=5, fill=X) + # Body. + frame_window.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) + frame_editor.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) # frame_run. + frame_run.pack(side=TOP, padx=5, pady=0, fill=X) startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) self.startup_shell_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) self.startup_editor_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - # frame_save. - run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) # frame_win_size. + frame_win_size.pack(side=TOP, padx=5, pady=0, fill=X) win_size_title.pack(side=LEFT, anchor=W, padx=5, pady=5) self.win_height_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) win_height_title.pack(side=RIGHT, anchor=E, pady=5) self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) win_width_title.pack(side=RIGHT, anchor=E, pady=5) + # frame_save. + frame_save.pack(side=TOP, padx=5, pady=0, fill=X) + run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) # frame_help. frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) diff --git a/Misc/NEWS.d/next/IDLE/2017-08-30-00-06-58.bpo-31051.50Jp_Q.rst b/Misc/NEWS.d/next/IDLE/2017-08-30-00-06-58.bpo-31051.50Jp_Q.rst new file mode 100644 index 00000000000..fe515911b0d --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2017-08-30-00-06-58.bpo-31051.50Jp_Q.rst @@ -0,0 +1 @@ +Rearrange IDLE condigdialog GenPage into Window, Editor, and Help sections.

1 0

bpo-5001: More-informative multiprocessing error messages (#3079)
by Antoine Pitrou 29 Aug '17

29 Aug '17

https://github.com/python/cpython/commit/bd73e72b4a9f019be514954b1d40e64dc3… commit: bd73e72b4a9f019be514954b1d40e64dc3a5e81c branch: master author: Allen W. Smith, Ph.D <drallensmith(a)users.noreply.github.com> committer: Antoine Pitrou <pitrou(a)free.fr> date: 2017-08-30T00:52:18+02:00 summary: bpo-5001: More-informative multiprocessing error messages (#3079) * Make error message more informative Replace assertions in error-reporting code with more-informative version that doesn't cause confusion over where and what the error is. * Additional clarification + get travis to check * Change from SystemError to TypeError As suggested in PR comment by @pitrou, changing from SystemError; TypeError appears appropriate. * NEWS file installation; ACKS addition (will do my best to justify it by additional work) * Making current AssertionErrors in multiprocessing more informative * Blurb added re multiprocessing managers.py, queues.py cleanup * Further multiprocessing cleanup - went through pool.py * Fix two asserts in multiprocessing/util.py * Most asserts in multiprocessing more informative * Didn't save right version * Further work on multiprocessing error messages * Correct typo * Correct typo v2 * Blasted colon... serves me right for trying to work on two things at once * Simplify NEWS entry * Update 2017-08-18-17-16-38.bpo-5001.gwnthq.rst * Update 2017-08-18-17-16-38.bpo-5001.gwnthq.rst OK, never mind. * Corrected (thanks to pitrou) error messages for notify * Remove extraneous backslash in docstring. files: A Misc/NEWS.d/next/Library/2017-08-18-17-16-38.bpo-5001.gwnthq.rst M Lib/multiprocessing/connection.py M Lib/multiprocessing/dummy/__init__.py M Lib/multiprocessing/forkserver.py M Lib/multiprocessing/heap.py M Lib/multiprocessing/managers.py M Lib/multiprocessing/pool.py M Lib/multiprocessing/popen_fork.py M Lib/multiprocessing/queues.py M Lib/multiprocessing/reduction.py M Lib/multiprocessing/resource_sharer.py M Lib/multiprocessing/semaphore_tracker.py M Lib/multiprocessing/spawn.py M Lib/multiprocessing/synchronize.py M Lib/multiprocessing/util.py diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index ba9b17cee1b..7a621a55f46 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -720,7 +720,9 @@ def PipeClient(address): def deliver_challenge(connection, authkey): import hmac - assert isinstance(authkey, bytes) + if not isinstance(authkey, bytes): + raise ValueError( + "Authkey must be bytes, not {0!s}".format(type(authkey))) message = os.urandom(MESSAGE_LENGTH) connection.send_bytes(CHALLENGE + message) digest = hmac.new(authkey, message, 'md5').digest() @@ -733,7 +735,9 @@ def deliver_challenge(connection, authkey): def answer_challenge(connection, authkey): import hmac - assert isinstance(authkey, bytes) + if not isinstance(authkey, bytes): + raise ValueError( + "Authkey must be bytes, not {0!s}".format(type(authkey))) message = connection.recv_bytes(256) # reject large message assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message message = message[len(CHALLENGE):] diff --git a/Lib/multiprocessing/dummy/__init__.py b/Lib/multiprocessing/dummy/__init__.py index cbb7f4909cb..403f5e5198e 100644 --- a/Lib/multiprocessing/dummy/__init__.py +++ b/Lib/multiprocessing/dummy/__init__.py @@ -41,7 +41,10 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): self._parent = current_process() def start(self): - assert self._parent is current_process() + if self._parent is not current_process(): + raise RuntimeError( + "Parent is {0!r} but current_process is {1!r}".format( + self._parent, current_process())) self._start_called = True if hasattr(self._parent, '_children'): self._parent._children[self] = None diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 69b842aa939..7a952e2ba69 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -189,7 +189,7 @@ def sigchld_handler(*_unused): if alive_r in rfds: # EOF because no more client processes left - assert os.read(alive_r, 1) == b'' + assert os.read(alive_r, 1) == b'', "Not at EOF?" raise SystemExit if sig_r in rfds: @@ -208,7 +208,10 @@ def sigchld_handler(*_unused): if os.WIFSIGNALED(sts): returncode = -os.WTERMSIG(sts) else: - assert os.WIFEXITED(sts) + if not os.WIFEXITED(sts): + raise AssertionError( + "Child {0:n} status is {1:n}".format( + pid,sts)) returncode = os.WEXITSTATUS(sts) # Send exit code to client process try: @@ -227,7 +230,10 @@ def sigchld_handler(*_unused): with listener.accept()[0] as s: # Receive fds from client fds = reduction.recvfds(s, MAXFDS_TO_SEND + 1) - assert len(fds) <= MAXFDS_TO_SEND + if len(fds) > MAXFDS_TO_SEND: + raise RuntimeError( + "Too many ({0:n}) fds to send".format( + len(fds))) child_r, child_w, *fds = fds s.close() pid = os.fork() diff --git a/Lib/multiprocessing/heap.py b/Lib/multiprocessing/heap.py index ee3ed551d0c..566173a1b0a 100644 --- a/Lib/multiprocessing/heap.py +++ b/Lib/multiprocessing/heap.py @@ -211,7 +211,10 @@ def free(self, block): # synchronously sometimes later from malloc() or free(), by calling # _free_pending_blocks() (appending and retrieving from a list is not # strictly thread-safe but under cPython it's atomic thanks to the GIL). - assert os.getpid() == self._lastpid + if os.getpid() != self._lastpid: + raise ValueError( + "My pid ({0:n}) is not last pid {1:n}".format( + os.getpid(),self._lastpid)) if not self._lock.acquire(False): # can't acquire the lock right now, add the block to the list of # pending blocks to free @@ -227,7 +230,10 @@ def free(self, block): def malloc(self, size): # return a block of right size (possibly rounded up) - assert 0 <= size < sys.maxsize + if size < 0: + raise ValueError("Size {0:n} out of range".format(size)) + if sys.maxsize <= size: + raise OverflowError("Size {0:n} too large".format(size)) if os.getpid() != self._lastpid: self.__init__() # reinitialize after fork with self._lock: @@ -250,7 +256,10 @@ class BufferWrapper(object): _heap = Heap() def __init__(self, size): - assert 0 <= size < sys.maxsize + if size < 0: + raise ValueError("Size {0:n} out of range".format(size)) + if sys.maxsize <= size: + raise OverflowError("Size {0:n} too large".format(size)) block = BufferWrapper._heap.malloc(size) self._state = (block, size) util.Finalize(self, BufferWrapper._heap.free, args=(block,)) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index c6722771b05..04df26bac66 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -23,7 +23,7 @@ from traceback import format_exc from . import connection -from .context import reduction, get_spawning_popen +from .context import reduction, get_spawning_popen, ProcessError from . import pool from . import process from . import util @@ -133,7 +133,10 @@ class Server(object): 'debug_info', 'number_of_objects', 'dummy', 'incref', 'decref'] def __init__(self, registry, address, authkey, serializer): - assert isinstance(authkey, bytes) + if not isinstance(authkey, bytes): + raise TypeError( + "Authkey {0!r} is type {1!s}, not bytes".format( + authkey, type(authkey))) self.registry = registry self.authkey = process.AuthenticationString(authkey) Listener, Client = listener_client[serializer] @@ -163,7 +166,7 @@ def serve_forever(self): except (KeyboardInterrupt, SystemExit): pass finally: - if sys.stdout != sys.__stdout__: + if sys.stdout != sys.__stdout__: # what about stderr? util.debug('resetting stdout, stderr') sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ @@ -316,6 +319,7 @@ def debug_info(self, c): ''' Return some info --- useful to spot problems with refcounting ''' + # Perhaps include debug info about 'c'? with self.mutex: result = [] keys = list(self.id_to_refcount.keys()) @@ -356,7 +360,9 @@ def create(self, c, typeid, *args, **kwds): self.registry[typeid] if callable is None: - assert len(args) == 1 and not kwds + if kwds or (len(args) != 1): + raise ValueError( + "Without callable, must have one non-keyword argument") obj = args[0] else: obj = callable(*args, **kwds) @@ -364,7 +370,10 @@ def create(self, c, typeid, *args, **kwds): if exposed is None: exposed = public_methods(obj) if method_to_typeid is not None: - assert type(method_to_typeid) is dict + if not isinstance(method_to_typeid, dict): + raise TypeError( + "Method_to_typeid {0!r}: type {1!s}, not dict".format( + method_to_typeid, type(method_to_typeid))) exposed = list(exposed) + list(method_to_typeid) ident = '%x' % id(obj) # convert to string because xmlrpclib @@ -417,7 +426,11 @@ def decref(self, c, ident): return with self.mutex: - assert self.id_to_refcount[ident] >= 1 + if self.id_to_refcount[ident] <= 0: + raise AssertionError( + "Id {0!s} ({1!r}) has refcount {2:n}, not 1+".format( + ident, self.id_to_obj[ident], + self.id_to_refcount[ident])) self.id_to_refcount[ident] -= 1 if self.id_to_refcount[ident] == 0: del self.id_to_refcount[ident] @@ -480,7 +493,14 @@ def get_server(self): ''' Return server object with serve_forever() method and address attribute ''' - assert self._state.value == State.INITIAL + if self._state.value != State.INITIAL: + if self._state.value == State.STARTED: + raise ProcessError("Already started server") + elif self._state.value == State.SHUTDOWN: + raise ProcessError("Manager has shut down") + else: + raise ProcessError( + "Unknown state {!r}".format(self._state.value)) return Server(self._registry, self._address, self._authkey, self._serializer) @@ -497,7 +517,14 @@ def start(self, initializer=None, initargs=()): ''' Spawn a server process for this manager object ''' - assert self._state.value == State.INITIAL + if self._state.value != State.INITIAL: + if self._state.value == State.STARTED: + raise ProcessError("Already started server") + elif self._state.value == State.SHUTDOWN: + raise ProcessError("Manager has shut down") + else: + raise ProcessError( + "Unknown state {!r}".format(self._state.value)) if initializer is not None and not callable(initializer): raise TypeError('initializer must be a callable') @@ -593,7 +620,14 @@ def _number_of_objects(self): def __enter__(self): if self._state.value == State.INITIAL: self.start() - assert self._state.value == State.STARTED + if self._state.value != State.STARTED: + if self._state.value == State.INITIAL: + raise ProcessError("Unable to start server") + elif self._state.value == State.SHUTDOWN: + raise ProcessError("Manager has shut down") + else: + raise ProcessError( + "Unknown state {!r}".format(self._state.value)) return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -653,7 +687,7 @@ def register(cls, typeid, callable=None, proxytype=None, exposed=None, getattr(proxytype, '_method_to_typeid_', None) if method_to_typeid: - for key, value in list(method_to_typeid.items()): + for key, value in list(method_to_typeid.items()): # isinstance? assert type(key) is str, '%r is not a string' % key assert type(value) is str, '%r is not a string' % value diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index c2364ab186c..e457f0a576f 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -92,7 +92,9 @@ def __repr__(self): def worker(inqueue, outqueue, initializer=None, initargs=(), maxtasks=None, wrap_exception=False): - assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0) + if (maxtasks is not None) and not (isinstance(maxtasks, int) + and maxtasks >= 1): + raise AssertionError("Maxtasks {!r} is not valid".format(maxtasks)) put = outqueue.put get = inqueue.get if hasattr(inqueue, '_writer'): @@ -254,8 +256,8 @@ def _setup_queues(self): def apply(self, func, args=(), kwds={}): ''' Equivalent of `func(*args, **kwds)`. + Pool must be running. ''' - assert self._state == RUN return self.apply_async(func, args, kwds).get() def map(self, func, iterable, chunksize=None): @@ -307,6 +309,10 @@ def imap(self, func, iterable, chunksize=1): )) return result else: + if chunksize < 1: + raise ValueError( + "Chunksize must be 1+, not {0:n}".format( + chunksize)) assert chunksize > 1 task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapIterator(self._cache) @@ -334,7 +340,9 @@ def imap_unordered(self, func, iterable, chunksize=1): )) return result else: - assert chunksize > 1 + if chunksize < 1: + raise ValueError( + "Chunksize must be 1+, not {0!r}".format(chunksize)) task_batches = Pool._get_tasks(func, iterable, chunksize) result = IMapUnorderedIterator(self._cache) self._taskqueue.put( @@ -466,7 +474,7 @@ def _handle_results(outqueue, get, cache): return if thread._state: - assert thread._state == TERMINATE + assert thread._state == TERMINATE, "Thread not in TERMINATE" util.debug('result handler found thread._state=TERMINATE') break @@ -542,7 +550,10 @@ def terminate(self): def join(self): util.debug('joining pool') - assert self._state in (CLOSE, TERMINATE) + if self._state == RUN: + raise ValueError("Pool is still running") + elif self._state not in (CLOSE, TERMINATE): + raise ValueError("In unknown state") self._worker_handler.join() self._task_handler.join() self._result_handler.join() @@ -570,7 +581,9 @@ def _terminate_pool(cls, taskqueue, inqueue, outqueue, pool, util.debug('helping task handler/workers to finish') cls._help_stuff_finish(inqueue, task_handler, len(pool)) - assert result_handler.is_alive() or len(cache) == 0 + if (not result_handler.is_alive()) and (len(cache) != 0): + raise AssertionError( + "Cannot have cache with result_hander not alive") result_handler._state = TERMINATE outqueue.put(None) # sentinel @@ -628,7 +641,8 @@ def ready(self): return self._event.is_set() def successful(self): - assert self.ready() + if not self.ready(): + raise ValueError("{0!r} not ready".format(self)) return self._success def wait(self, timeout=None): diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 44ce9a9e79b..cbdbfa79cf3 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -35,7 +35,7 @@ def poll(self, flag=os.WNOHANG): if os.WIFSIGNALED(sts): self.returncode = -os.WTERMSIG(sts) else: - assert os.WIFEXITED(sts) + assert os.WIFEXITED(sts), "Status is {:n}".format(sts) self.returncode = os.WEXITSTATUS(sts) return self.returncode diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 513807cafec..328efbd95fe 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -78,7 +78,7 @@ def _after_fork(self): self._poll = self._reader.poll def put(self, obj, block=True, timeout=None): - assert not self._closed + assert not self._closed, "Queue {0!r} has been closed".format(self) if not self._sem.acquire(block, timeout): raise Full @@ -140,7 +140,7 @@ def close(self): def join_thread(self): debug('Queue.join_thread()') - assert self._closed + assert self._closed, "Queue {0!r} not closed".format(self) if self._jointhread: self._jointhread() @@ -281,7 +281,7 @@ def __setstate__(self, state): self._cond, self._unfinished_tasks = state[-2:] def put(self, obj, block=True, timeout=None): - assert not self._closed + assert not self._closed, "Queue {0!r} is closed".format(self) if not self._sem.acquire(block, timeout): raise Full diff --git a/Lib/multiprocessing/reduction.py b/Lib/multiprocessing/reduction.py index 7f65947379f..deca19ccad7 100644 --- a/Lib/multiprocessing/reduction.py +++ b/Lib/multiprocessing/reduction.py @@ -165,7 +165,10 @@ def recvfds(sock, size): if len(cmsg_data) % a.itemsize != 0: raise ValueError a.frombytes(cmsg_data) - assert len(a) % 256 == msg[0] + if len(a) % 256 != msg[0]: + raise AssertionError( + "Len is {0:n} but msg[0] is {1!r}".format( + len(a), msg[0])) return list(a) except (ValueError, IndexError): pass diff --git a/Lib/multiprocessing/resource_sharer.py b/Lib/multiprocessing/resource_sharer.py index e44a728fa9a..6d99da102ff 100644 --- a/Lib/multiprocessing/resource_sharer.py +++ b/Lib/multiprocessing/resource_sharer.py @@ -125,7 +125,7 @@ def _afterfork(self): def _start(self): from .connection import Listener - assert self._listener is None + assert self._listener is None, "Already have Listener" util.debug('starting listener and thread for sending handles') self._listener = Listener(authkey=process.current_process().authkey) self._address = self._listener.address diff --git a/Lib/multiprocessing/semaphore_tracker.py b/Lib/multiprocessing/semaphore_tracker.py index de7738eeee8..d5f259c246b 100644 --- a/Lib/multiprocessing/semaphore_tracker.py +++ b/Lib/multiprocessing/semaphore_tracker.py @@ -80,7 +80,8 @@ def _send(self, cmd, name): # bytes are atomic, and that PIPE_BUF >= 512 raise ValueError('name too long') nbytes = os.write(self._fd, msg) - assert nbytes == len(msg) + assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( + nbytes, len(msg)) _semaphore_tracker = SemaphoreTracker() diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index 4aba372e48e..1f4f3f496f5 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -93,7 +93,7 @@ def spawn_main(pipe_handle, parent_pid=None, tracker_fd=None): ''' Run code specified by data received over pipe ''' - assert is_forking(sys.argv) + assert is_forking(sys.argv), "Not forking" if sys.platform == 'win32': import msvcrt new_handle = reduction.steal_handle(parent_pid, pipe_handle) diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 0590ed68f53..038f73f6b76 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -270,13 +270,16 @@ def wait(self, timeout=None): def notify(self, n=1): assert self._lock._semlock._is_mine(), 'lock is not owned' - assert not self._wait_semaphore.acquire(False) + assert not self._wait_semaphore.acquire( + False), ('notify: Should not have been able to acquire' + + '_wait_semaphore') # to take account of timeouts since last notify*() we subtract # woken_count from sleeping_count and rezero woken_count while self._woken_count.acquire(False): res = self._sleeping_count.acquire(False) - assert res + assert res, ('notify: Bug in sleeping_count.acquire' + + '- res should not be False') sleepers = 0 while sleepers < n and self._sleeping_count.acquire(False): diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index b490caa7e64..f0827f0a969 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -149,12 +149,15 @@ class Finalize(object): Class which supports object finalization using weakrefs ''' def __init__(self, obj, callback, args=(), kwargs=None, exitpriority=None): - assert exitpriority is None or type(exitpriority) is int + if (exitpriority is not None) and not isinstance(exitpriority,int): + raise TypeError( + "Exitpriority ({0!r}) must be None or int, not {1!s}".format( + exitpriority, type(exitpriority))) if obj is not None: self._weakref = weakref.ref(obj, self) - else: - assert exitpriority is not None + elif exitpriority is None: + raise ValueError("Without object, exitpriority cannot be None") self._callback = callback self._args = args diff --git a/Misc/NEWS.d/next/Library/2017-08-18-17-16-38.bpo-5001.gwnthq.rst b/Misc/NEWS.d/next/Library/2017-08-18-17-16-38.bpo-5001.gwnthq.rst new file mode 100644 index 00000000000..640b801c9b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-08-18-17-16-38.bpo-5001.gwnthq.rst @@ -0,0 +1 @@ +Many asserts in `multiprocessing` are now more informative, and some error types have been changed to more specific ones.

1 0

[65 flat] Results for Python (master branch) 2017-08-29
by lp_benchmark_robot＠intel.com 29 Aug '17

29 Aug '17

Results for project python/master, build date: 2017-08-29 03:04:28-07:00. - commit: ba7d736 - previous commit: a30f6d4 - revision date: 2017-08-29 11:58:27+03:00 - environment: Broadwell-EP - cpu: Intel(R) Xeon(R) CPU E5-2699 v4 @ 2.20GHz 2x22 cores, stepping 1, LLC 55 MB - mem: 128 GB - os: Ubuntu 16.04.2 LTS - kernel: 4.4.0-62-generic x86_64 GNU/Linux Baseline results were generated using release v3.6.0, with hash 5c4568a from 2016-12-22 23:38:47+00:00. +-----+------------------------+--------+------------+------------+------------+ | | |relative|change since|change since|current rev | | | benchmark|std_dev*| last run | baseline |run with PGO| +-----+------------------------+--------+------------+------------+------------+ | :-| | 2to3| 1.020% | +1.174% | +4.348% | +7.328% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method| 1.898% | +0.045% | +22.982% | +10.105% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_slots| 2.010% | -0.229% | +23.675% | +11.233% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_method_unknown| 0.388% | -0.057% | +22.017% | +8.866% | +-----+------------------------+--------+------------+------------+------------+ | :-| | call_simple| 2.252% | +0.215% | +3.163% | +13.426% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chameleon| 2.551% | +0.254% | +12.084% | +6.789% | +-----+------------------------+--------+------------+------------+------------+ | :-| | chaos| 1.598% | +0.288% | +6.172% | +11.219% | +-----+------------------------+--------+------------+------------+------------+ | :-| | crypto_pyaes| 0.496% | +0.081% | +3.282% | +5.731% | +-----+------------------------+--------+------------+------------+------------+ | :-| | deltablue| 4.639% | -0.384% | +6.108% | +19.248% | +-----+------------------------+--------+------------+------------+------------+ | :-| | django_template| 7.423% | +1.798% | +6.440% | +10.361% | +-----+------------------------+--------+------------+------------+------------+ | :-| | dulwich_log| 1.084% | +1.363% | +4.563% | +6.881% | +-----+------------------------+--------+------------+------------+------------+ | :-| | fannkuch| 0.460% | +0.547% | +5.535% | +2.588% | +-----+------------------------+--------+------------+------------+------------+ | :-| | float| 0.941% | +0.102% | +3.593% | +5.871% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_text| 1.498% | +0.364% | +8.091% | +10.837% | +-----+------------------------+--------+------------+------------+------------+ | :-| | genshi_xml| 2.020% | +1.758% | +6.907% | +8.155% | +-----+------------------------+--------+------------+------------+------------+ | :-| | go| 1.028% | +0.190% | +6.350% | +9.823% | +-----+------------------------+--------+------------+------------+------------+ | :-| | hexiom| 2.023% | -0.174% | +9.268% | +10.801% | +-----+------------------------+--------+------------+------------+------------+ | :-| | html5lib| 3.577% | +2.060% | +6.706% | +9.433% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_dumps| 2.873% | +0.731% | +4.167% | +8.247% | +-----+------------------------+--------+------------+------------+------------+ | :-| | json_loads| 4.272% | +0.681% | +1.787% | +8.028% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_format| 1.705% | +3.116% | +7.335% | +10.787% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_silent| 2.582% | -0.129% | +46.685% | +12.357% | +-----+------------------------+--------+------------+------------+------------+ | :-| | logging_simple| 1.742% | +3.006% | +7.929% | +11.733% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mako| 0.379% | +0.183% | +19.151% | +11.157% | +-----+------------------------+--------+------------+------------+------------+ | :-| | mdp| 5.274% | +0.652% | +6.281% | +12.484% | +-----+------------------------+--------+------------+------------+------------+ | :-| | meteor_contest| 2.603% | -0.006% | +4.416% | +4.171% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nbody| 0.918% | -0.011% | -3.926% | +4.561% | +-----+------------------------+--------+------------+------------+------------+ | :-| | nqueens| 0.702% | -0.168% | +2.922% | +7.050% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pathlib| 1.174% | +2.490% | +1.939% | +10.537% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle| 2.674% | +0.947% | +0.301% | +22.780% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_dict| 0.343% | +0.088% | +2.572% | +19.991% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_list| 0.806% | -0.049% | +6.582% | +18.419% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pickle_pure_python| 1.828% | +2.773% | +12.421% | +9.048% | +-----+------------------------+--------+------------+------------+------------+ | :-| | pidigits| 0.159% | +0.082% | +0.429% | +10.092% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup| 0.131% | +0.570% | +8.759% | +4.360% | +-----+------------------------+--------+------------+------------+------------+ | :-| | python_startup_no_site| 0.103% | +0.548% | +0.901% | +4.225% | +-----+------------------------+--------+------------+------------+------------+ | :-| | raytrace| 1.423% | +0.508% | +8.752% | +14.207% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_compile| 5.310% | -0.471% | -8.909% | +10.681% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_dna| 1.076% | +0.018% | +2.266% | +8.075% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_effbot| 1.915% | +0.234% | +0.500% | -0.106% | +-----+------------------------+--------+------------+------------+------------+ | :-| | regex_v8| 0.739% | +0.065% | +12.042% | +2.856% | +-----+------------------------+--------+------------+------------+------------+ | :-| | richards| 4.440% | -0.740% | +6.701% | +15.955% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_fft| 0.629% | +0.261% | +1.326% | -0.684% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_lu| 2.566% | +0.607% | +25.980% | +8.772% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_monte_carlo| 1.334% | -0.265% | +2.840% | +7.908% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sor| 0.479% | +0.030% | +14.391% | +8.504% | +-----+------------------------+--------+------------+------------+------------+ | :-| | scimark_sparse_mat_mult| 0.935% | -0.135% | +0.952% | -3.934% | +-----+------------------------+--------+------------+------------+------------+ | :-| | spectral_norm| 0.804% | +0.469% | +5.760% | +1.052% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_declarative| 1.257% | +0.292% | +4.713% | +7.463% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlalchemy_imperative| 2.809% | +0.686% | +4.999% | +4.861% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sqlite_synth| 4.084% | +2.697% | -2.233% | +12.523% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_expand| 2.256% | +1.039% | +12.213% | +7.276% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_integrate| 2.182% | +0.485% | +9.736% | +6.187% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_str| 3.924% | +1.083% | +12.041% | +7.361% | +-----+------------------------+--------+------------+------------+------------+ | :-| | sympy_sum| 4.814% | +1.751% | +13.341% | +8.097% | +-----+------------------------+--------+------------+------------+------------+ | :-| | telco| 4.431% | +0.953% | +21.692% | +12.694% | +-----+------------------------+--------+------------+------------+------------+ | :-| | tornado_http| 1.215% | +0.857% | +5.552% | +6.515% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpack_sequence| 1.074% | -0.172% | +2.577% | -1.516% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle|10.391% | -3.098% | +6.260% | +22.447% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_list| 2.107% | -0.449% | -3.069% | +20.594% | +-----+------------------------+--------+------------+------------+------------+ | :-| | unpickle_pure_python| 1.311% | +0.699% | +7.018% | +6.583% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_generate| 1.057% | +0.218% | +6.143% | +6.329% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_iterparse| 2.143% | +0.049% | +3.546% | +4.424% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_parse| 2.004% | -0.584% | -4.932% | +9.499% | +-----+------------------------+--------+------------+------------+------------+ | :-| | xml_etree_process| 1.356% | +0.518% | +7.223% | +6.732% | +-----+------------------------+--------+------------+------------+------------+ * Relative Standard Deviation (Standard Deviation/Average) If this is not displayed properly please visit our results page here: http://languagesperformance.intel.com/65-flat-results-for-python-master-bra… Our lab does a nightly source pull and build of the Python project and measures performance changes against the previous stable version and the previous nightly measurement. This is provided as a service to the community so that quality issues with current hardware can be identified quickly. Intel technologies' features and benefits depend on system configuration and may require enabled hardware, software or service activation. Performance varies depending on system configuration.

1 0

bpo-31291: Fixed an assertion failure in zipimport.zipimporter.get_data() (#3226)
by Serhiy Storchaka 29 Aug '17

29 Aug '17

https://github.com/python/cpython/commit/631fdee6e61b4ba8ce800f827fecdd536b… commit: 631fdee6e61b4ba8ce800f827fecdd536bfb04f3 branch: master author: Oren Milman <orenmn(a)gmail.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-08-29T20:40:15+03:00 summary: bpo-31291: Fixed an assertion failure in zipimport.zipimporter.get_data() (#3226) if pathname.replace('/', '\\') returns non-string. files: A Misc/NEWS.d/next/Core and Builtins/2017-08-28-11-51-29.bpo-31291.t8QggK.rst M Lib/test/test_zipimport.py M Modules/zipimport.c diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 7ddbc509f9b..daa5138751b 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -521,6 +521,23 @@ def testGetData(self): z.close() os.remove(TEMP_ZIP) + def test_issue31291(self): + # There shouldn't be an assertion failure in get_data(). + class FunnyStr(str): + def replace(self, old, new): + return 42 + z = ZipFile(TEMP_ZIP, "w") + try: + name = "test31291.dat" + data = b'foo' + z.writestr(name, data) + z.close() + zi = zipimport.zipimporter(TEMP_ZIP) + self.assertEqual(data, zi.get_data(FunnyStr(name))) + finally: + z.close() + os.remove(TEMP_ZIP) + def testImporterAttr(self): src = """if 1: # indent hack def get_file(): diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-08-28-11-51-29.bpo-31291.t8QggK.rst b/Misc/NEWS.d/next/Core and Builtins/2017-08-28-11-51-29.bpo-31291.t8QggK.rst new file mode 100644 index 00000000000..0576785c7df --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-08-28-11-51-29.bpo-31291.t8QggK.rst @@ -0,0 +1,3 @@ +Fix an assertion failure in `zipimport.zipimporter.get_data` on Windows, +when the return value of ``pathname.replace('/','\\')`` isn't a string. +Patch by Oren Milman. diff --git a/Modules/zipimport.c b/Modules/zipimport.c index c01ee55840b..d710360f95d 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -651,7 +651,8 @@ zipimport_zipimporter_get_data_impl(ZipImporter *self, PyObject *path) Py_ssize_t path_start, path_len, len; #ifdef ALTSEP - path = _PyObject_CallMethodId(path, &PyId_replace, "CC", ALTSEP, SEP); + path = _PyObject_CallMethodId((PyObject *)&PyUnicode_Type, &PyId_replace, + "OCC", path, ALTSEP, SEP); if (!path) return NULL; #else

1 0

[2.7] bpo-31243: Fixed PyArg_ParseTuple failure checks. (GH-3171) (#3235)
by Serhiy Storchaka 29 Aug '17

29 Aug '17

https://github.com/python/cpython/commit/20958e6d91d11a80d6c664ce6346791259… commit: 20958e6d91d11a80d6c664ce6346791259b1d193 branch: 2.7 author: Oren Milman <orenmn(a)gmail.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2017-08-29T19:16:12+03:00 summary: [2.7] bpo-31243: Fixed PyArg_ParseTuple failure checks. (GH-3171) (#3235) files: A Misc/NEWS.d/next/Core and Builtins/2017-08-29-14-24-34.bpo-31243.tr0E4V.rst M Lib/test/test_io.py M Modules/_io/textio.c M Modules/_testcapimodule.c diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index caca0333e0d..3dc6e9ffcac 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2702,6 +2702,26 @@ def _make_illegal_wrapper(): #t = _make_illegal_wrapper() #self.assertRaises(TypeError, t.read) + # Issue 31243: calling read() while the return value of decoder's + # getstate() is invalid should neither crash the interpreter nor + # raise a SystemError. + def _make_very_illegal_wrapper(getstate_ret_val): + class BadDecoder: + def getstate(self): + return getstate_ret_val + def _get_bad_decoder(dummy): + return BadDecoder() + quopri = codecs.lookup("quopri_codec") + with support.swap_attr(quopri, 'incrementaldecoder', + _get_bad_decoder): + return _make_illegal_wrapper() + t = _make_very_illegal_wrapper(42) + with self.maybeRaises(TypeError): + t.read(42) + t = _make_very_illegal_wrapper(()) + with self.maybeRaises(TypeError): + t.read(42) + class CTextIOWrapperTest(TextIOWrapperTest): diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-08-29-14-24-34.bpo-31243.tr0E4V.rst b/Misc/NEWS.d/next/Core and Builtins/2017-08-29-14-24-34.bpo-31243.tr0E4V.rst new file mode 100644 index 00000000000..166458f2b78 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-08-29-14-24-34.bpo-31243.tr0E4V.rst @@ -0,0 +1,2 @@ +Fix a crash in some methods of `io.TextIOWrapper`, when the decoder's state +is invalid. Patch by Oren Milman. diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index f13dcb4366e..d9e036db386 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1416,7 +1416,7 @@ textiowrapper_read_chunk(textio *self) /* Given this, we know there was a valid snapshot point * len(dec_buffer) bytes ago with decoder state (b'', dec_flags). */ - if (PyArg_Parse(state, "(OO)", &dec_buffer, &dec_flags) < 0) { + if (!PyArg_Parse(state, "(OO)", &dec_buffer, &dec_flags)) { Py_DECREF(state); return -1; } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a5e3f07667d..ecc473cf4c1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -911,8 +911,9 @@ test_L_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, num); value = -1; - if (PyArg_ParseTuple(tuple, "L:test_L_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { return NULL; + } if (value != 42) return raiseTestError("test_L_code", "L code returned wrong value for long 42"); @@ -925,8 +926,9 @@ test_L_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, num); value = -1; - if (PyArg_ParseTuple(tuple, "L:test_L_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { return NULL; + } if (value != 42) return raiseTestError("test_L_code", "L code returned wrong value for int 42"); @@ -1207,8 +1209,9 @@ test_k_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, num); value = 0; - if (PyArg_ParseTuple(tuple, "k:test_k_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { return NULL; + } if (value != ULONG_MAX) return raiseTestError("test_k_code", "k code returned wrong value for long 0xFFF...FFF"); @@ -1226,8 +1229,9 @@ test_k_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, num); value = 0; - if (PyArg_ParseTuple(tuple, "k:test_k_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { return NULL; + } if (value != (unsigned long)-0x42) return raiseTestError("test_k_code", "k code returned wrong value for long -0xFFF..000042"); @@ -1587,14 +1591,16 @@ test_u_code(PyObject *self) PyTuple_SET_ITEM(tuple, 0, obj); value = 0; - if (PyArg_ParseTuple(tuple, "u:test_u_code", &value) < 0) + if (!PyArg_ParseTuple(tuple, "u:test_u_code", &value)) { return NULL; + } if (value != PyUnicode_AS_UNICODE(obj)) return raiseTestError("test_u_code", "u code returned wrong value for u'test'"); value = 0; - if (PyArg_ParseTuple(tuple, "u#:test_u_code", &value, &len) < 0) + if (!PyArg_ParseTuple(tuple, "u#:test_u_code", &value, &len)) { return NULL; + } if (value != PyUnicode_AS_UNICODE(obj) || len != PyUnicode_GET_SIZE(obj)) return raiseTestError("test_u_code", @@ -1692,8 +1698,9 @@ test_empty_argparse(PyObject *self) tuple = PyTuple_New(0); if (!tuple) return NULL; - if ((result = PyArg_ParseTuple(tuple, "|:test_empty_argparse")) < 0) + if (!(result = PyArg_ParseTuple(tuple, "|:test_empty_argparse"))) { goto done; + } dict = PyDict_New(); if (!dict) goto done; @@ -1701,8 +1708,9 @@ test_empty_argparse(PyObject *self) done: Py_DECREF(tuple); Py_XDECREF(dict); - if (result < 0) + if (!result) { return NULL; + } else { Py_RETURN_NONE; } @@ -2494,8 +2502,9 @@ test_raise_signal(PyObject* self, PyObject *args) { int signum, err; - if (PyArg_ParseTuple(args, "i:raise_signal", &signum) < 0) + if (!PyArg_ParseTuple(args, "i:raise_signal", &signum)) { return NULL; + } err = raise(signum); if (err)

1 0

bpo-31065: Add doc about Popen.poll returning None. (#3169)
by Gregory P. Smith 29 Aug '17

29 Aug '17

https://github.com/python/cpython/commit/006617ff7d6df3fdedcfe53e94ee2c52cc… commit: 006617ff7d6df3fdedcfe53e94ee2c52cc796437 branch: master author: Ivan Chernoff <chernoffivan(a)gmail.com> committer: Gregory P. Smith <greg(a)krypto.org> date: 2017-08-29T07:46:24-07:00 summary: bpo-31065: Add doc about Popen.poll returning None. (#3169) files: M Doc/library/subprocess.rst diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 2eaa48e070d..a4e234e62ae 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -584,7 +584,7 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.poll() Check if child process has terminated. Set and return - :attr:`~Popen.returncode` attribute. + :attr:`~Popen.returncode` attribute. Otherwise, returns ``None``. .. method:: Popen.wait(timeout=None)

1 0

bpo-31286, bpo-30024: Fixed stack usage in absolute imports with (#3217)
by Serhiy Storchaka 29 Aug '17

29 Aug '17

https://github.com/python/cpython/commit/265fcc5fc25d65afb33fda480c603f1e97… commit: 265fcc5fc25d65afb33fda480c603f1e974e374e branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2017-08-29T15:47:44+03:00 summary: bpo-31286, bpo-30024: Fixed stack usage in absolute imports with (#3217) binding a submodule to a name. files: M Lib/test/test_import/__init__.py M Python/compile.c diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 1b04de24dd1..ddef27d9cd4 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -237,6 +237,23 @@ def test_import_name_binding(self): import test.support as y self.assertIs(y, test.support, y.__name__) + def test_issue31286(self): + # import in a 'finally' block resulted in SystemError + try: + x = ... + finally: + import test.support.script_helper as x + + # import in a 'while' loop resulted in stack overflow + i = 0 + while i < 10: + import test.support.script_helper as x + i += 1 + + # import in a 'for' loop resulted in segmentation fault + for i in range(2): + import test.support.script_helper as x + def test_failing_reload(self): # A failing reload should leave the module object in sys.modules. source = TESTFN + os.extsep + "py" diff --git a/Python/compile.c b/Python/compile.c index 78e797a2c33..e547c2fd591 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2673,28 +2673,34 @@ compiler_import_as(struct compiler *c, identifier name, identifier asname) If there is a dot in name, we need to split it and emit a IMPORT_FROM for each name. */ - Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, - PyUnicode_GET_LENGTH(name), 1); + Py_ssize_t len = PyUnicode_GET_LENGTH(name); + Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, len, 1); if (dot == -2) return 0; if (dot != -1) { /* Consume the base module name to get the first attribute */ - Py_ssize_t pos = dot + 1; - while (dot != -1) { + while (1) { + Py_ssize_t pos = dot + 1; PyObject *attr; - dot = PyUnicode_FindChar(name, '.', pos, - PyUnicode_GET_LENGTH(name), 1); + dot = PyUnicode_FindChar(name, '.', pos, len, 1); if (dot == -2) return 0; - attr = PyUnicode_Substring(name, pos, - (dot != -1) ? dot : - PyUnicode_GET_LENGTH(name)); + attr = PyUnicode_Substring(name, pos, (dot != -1) ? dot : len); if (!attr) return 0; ADDOP_O(c, IMPORT_FROM, attr, names); Py_DECREF(attr); - pos = dot + 1; + if (dot == -1) { + break; + } + ADDOP(c, ROT_TWO); + ADDOP(c, POP_TOP); } + if (!compiler_nameop(c, asname, Store)) { + return 0; + } + ADDOP(c, POP_TOP); + return 1; } return compiler_nameop(c, asname, Store); }

1 0