Mailman 3 January 2018 - Python-checkins

bpo-25942: make subprocess more graceful on ^C (GH-5026)
by Gregory P. Smith Jan. 30, 2018

Jan. 30, 2018

https://github.com/python/cpython/commit/f4d644f36ffb6cb11b34bfcf533c14cfae… commit: f4d644f36ffb6cb11b34bfcf533c14cfaebf709a branch: master author: Gregory P. Smith <greg(a)krypto.org> committer: GitHub <noreply(a)github.com> date: 2018-01-29T21:27:39-08:00 summary: bpo-25942: make subprocess more graceful on ^C (GH-5026) Do not allow receiving a SIGINT to cause the subprocess module to trigger an immediate SIGKILL of the child process. SIGINT is normally sent to all child processes by the OS at the same time already as was the established normal behavior in 2.7 and 3.2. This behavior change was introduced during the fix to https://bugs.python.org/issue12494 and is generally surprising to command line tool users who expect other tools launched in child processes to get their own SIGINT and do their own cleanup. In Python 3.3-3.6 subprocess.call and subprocess.run would immediately SIGKILL the child process upon receiving a SIGINT (which raises a KeyboardInterrupt). We now give the child a small amount of time to exit gracefully before resorting to a SIGKILL. This is also the case for subprocess.Popen.__exit__ which would previously block indefinitely waiting for the child to die. This was hidden from many users by virtue of subprocess.call and subprocess.run sending the signal immediately. Behavior change: subprocess.Popen.__exit__ will not block indefinitely when the exiting exception is a KeyboardInterrupt. This is done for user friendliness as people expect their ^C to actually happen. This could cause occasional orphaned Popen objects when not using `call` or `run` with a child process that hasn't exited. Refactoring involved: The Popen.wait method deals with the KeyboardInterrupt second chance, existing platform specific internals have been renamed to _wait(). Also fixes comment typos. files: A Misc/NEWS.d/next/Library/2017-12-27-20-15-51.bpo-25942.Giyr8v.rst M Lib/subprocess.py M Lib/test/test_subprocess.py diff --git a/Lib/subprocess.py b/Lib/subprocess.py index db6342fa4912..f69159e3aadc 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -304,9 +304,9 @@ def call(*popenargs, timeout=None, **kwargs): with Popen(*popenargs, **kwargs) as p: try: return p.wait(timeout=timeout) - except: + except: # Including KeyboardInterrupt, wait handled that. p.kill() - p.wait() + # We don't call p.wait() again as p.__exit__ does that for us. raise @@ -450,9 +450,9 @@ def run(*popenargs, input=None, timeout=None, check=False, **kwargs): stdout, stderr = process.communicate() raise TimeoutExpired(process.args, timeout, output=stdout, stderr=stderr) - except: + except: # Including KeyboardInterrupt, communicate handled that. process.kill() - process.wait() + # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: @@ -714,6 +714,11 @@ def __init__(self, args, bufsize=-1, executable=None, self.text_mode = encoding or errors or text or universal_newlines + # How long to resume waiting on a child after the first ^C. + # There is no right value for this. The purpose is to be polite + # yet remain good for interactive users trying to exit a tool. + self._sigint_wait_secs = 0.25 # 1/xkcd221.getRandomNumber() + self._closed_child_pipe_fds = False try: @@ -787,7 +792,7 @@ def _translate_newlines(self, data, encoding, errors): def __enter__(self): return self - def __exit__(self, type, value, traceback): + def __exit__(self, exc_type, value, traceback): if self.stdout: self.stdout.close() if self.stderr: @@ -796,6 +801,22 @@ def __exit__(self, type, value, traceback): if self.stdin: self.stdin.close() finally: + if exc_type == KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # In the case of a KeyboardInterrupt we assume the SIGINT + # was also already sent to our child processes. We can't + # block indefinitely as that is not user friendly. + # If we have not already waited a brief amount of time in + # an interrupted .wait() or .communicate() call, do so here + # for consistency. + if self._sigint_wait_secs > 0: + try: + self._wait(timeout=self._sigint_wait_secs) + except TimeoutExpired: + pass + self._sigint_wait_secs = 0 # Note that this has been done. + return # resume the KeyboardInterrupt + # Wait for the process to terminate, to avoid zombies. self.wait() @@ -804,7 +825,7 @@ def __del__(self, _maxsize=sys.maxsize, _warn=warnings.warn): # We didn't get to successfully create a child process. return if self.returncode is None: - # Not reading subprocess exit status creates a zombi process which + # Not reading subprocess exit status creates a zombie process which # is only destroyed at the parent python process exit _warn("subprocess %s is still running" % self.pid, ResourceWarning, source=self) @@ -889,6 +910,21 @@ def communicate(self, input=None, timeout=None): try: stdout, stderr = self._communicate(input, endtime, timeout) + except KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # See the detailed comment in .wait(). + if timeout is not None: + sigint_timeout = min(self._sigint_wait_secs, + self._remaining_time(endtime)) + else: + sigint_timeout = self._sigint_wait_secs + self._sigint_wait_secs = 0 # nothing else should wait. + try: + self._wait(timeout=sigint_timeout) + except TimeoutExpired: + pass + raise # resume the KeyboardInterrupt + finally: self._communication_started = True @@ -919,6 +955,30 @@ def _check_timeout(self, endtime, orig_timeout): raise TimeoutExpired(self.args, orig_timeout) + def wait(self, timeout=None): + """Wait for child process to terminate; returns self.returncode.""" + if timeout is not None: + endtime = _time() + timeout + try: + return self._wait(timeout=timeout) + except KeyboardInterrupt: + # https://bugs.python.org/issue25942 + # The first keyboard interrupt waits briefly for the child to + # exit under the common assumption that it also received the ^C + # generated SIGINT and will exit rapidly. + if timeout is not None: + sigint_timeout = min(self._sigint_wait_secs, + self._remaining_time(endtime)) + else: + sigint_timeout = self._sigint_wait_secs + self._sigint_wait_secs = 0 # nothing else should wait. + try: + self._wait(timeout=sigint_timeout) + except TimeoutExpired: + pass + raise # resume the KeyboardInterrupt + + if _mswindows: # # Windows methods @@ -1127,16 +1187,16 @@ def _internal_poll(self, _deadstate=None, return self.returncode - def wait(self, timeout=None): - """Wait for child process to terminate. Returns returncode - attribute.""" + def _wait(self, timeout): + """Internal implementation of wait() on Windows.""" if timeout is None: timeout_millis = _winapi.INFINITE else: timeout_millis = int(timeout * 1000) if self.returncode is None: + # API note: Returns immediately if timeout_millis == 0. result = _winapi.WaitForSingleObject(self._handle, - timeout_millis) + timeout_millis) if result == _winapi.WAIT_TIMEOUT: raise TimeoutExpired(self.args, timeout) self.returncode = _winapi.GetExitCodeProcess(self._handle) @@ -1498,9 +1558,8 @@ def _try_wait(self, wait_flags): return (pid, sts) - def wait(self, timeout=None): - """Wait for child process to terminate. Returns returncode - attribute.""" + def _wait(self, timeout): + """Internal implementation of wait() on POSIX.""" if self.returncode is not None: return self.returncode diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 2e2721e5e998..dd63818f3b2f 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2921,6 +2921,71 @@ def test_terminate_dead(self): self._kill_dead_process('terminate') class MiscTests(unittest.TestCase): + + class RecordingPopen(subprocess.Popen): + """A Popen that saves a reference to each instance for testing.""" + instances_created = [] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.instances_created.append(self) + + @mock.patch.object(subprocess.Popen, "_communicate") + def _test_keyboardinterrupt_no_kill(self, popener, mock__communicate, + **kwargs): + """Fake a SIGINT happening during Popen._communicate() and ._wait(). + + This avoids the need to actually try and get test environments to send + and receive signals reliably across platforms. The net effect of a ^C + happening during a blocking subprocess execution which we want to clean + up from is a KeyboardInterrupt coming out of communicate() or wait(). + """ + + mock__communicate.side_effect = KeyboardInterrupt + try: + with mock.patch.object(subprocess.Popen, "_wait") as mock__wait: + # We patch out _wait() as no signal was involved so the + # child process isn't actually going to exit rapidly. + mock__wait.side_effect = KeyboardInterrupt + with mock.patch.object(subprocess, "Popen", + self.RecordingPopen): + with self.assertRaises(KeyboardInterrupt): + popener([sys.executable, "-c", + "import time\ntime.sleep(9)\nimport sys\n" + "sys.stderr.write('\\n!runaway child!\\n')"], + stdout=subprocess.DEVNULL, **kwargs) + for call in mock__wait.call_args_list[1:]: + self.assertNotEqual( + call, mock.call(timeout=None), + "no open-ended wait() after the first allowed: " + f"{mock__wait.call_args_list}") + sigint_calls = [] + for call in mock__wait.call_args_list: + if call == mock.call(timeout=0.25): # from Popen.__init__ + sigint_calls.append(call) + self.assertLessEqual(mock__wait.call_count, 2, + msg=mock__wait.call_args_list) + self.assertEqual(len(sigint_calls), 1, + msg=mock__wait.call_args_list) + finally: + # cleanup the forgotten (due to our mocks) child process + process = self.RecordingPopen.instances_created.pop() + process.kill() + process.wait() + self.assertEqual([], self.RecordingPopen.instances_created) + + def test_call_keyboardinterrupt_no_kill(self): + self._test_keyboardinterrupt_no_kill(subprocess.call, timeout=6.282) + + def test_run_keyboardinterrupt_no_kill(self): + self._test_keyboardinterrupt_no_kill(subprocess.run, timeout=6.282) + + def test_context_manager_keyboardinterrupt_no_kill(self): + def popen_via_context_manager(*args, **kwargs): + with subprocess.Popen(*args, **kwargs) as unused_process: + raise KeyboardInterrupt # Test how __exit__ handles ^C. + self._test_keyboardinterrupt_no_kill(popen_via_context_manager) + def test_getoutput(self): self.assertEqual(subprocess.getoutput('echo xyzzy'), 'xyzzy') self.assertEqual(subprocess.getstatusoutput('echo xyzzy'), diff --git a/Misc/NEWS.d/next/Library/2017-12-27-20-15-51.bpo-25942.Giyr8v.rst b/Misc/NEWS.d/next/Library/2017-12-27-20-15-51.bpo-25942.Giyr8v.rst new file mode 100644 index 000000000000..b898345b2b32 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-27-20-15-51.bpo-25942.Giyr8v.rst @@ -0,0 +1,6 @@ +The subprocess module is now more graceful when handling a Ctrl-C +KeyboardInterrupt during subprocess.call, subprocess.run, or a Popen context +manager. It now waits a short amount of time for the child (presumed to +have also gotten the SIGINT) to exit, before continuing the +KeyboardInterrupt exception handling. This still includes a SIGKILL in the +call() and run() APIs, but at least the child had a chance first.

1 0

bpo-32701: Clarify the quotetabs flag in quopri documentation (GH-5401) (GH-5438)
by Mariatta Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/04f99ba9d7186278eaf072e9a62c03aa63… commit: 04f99ba9d7186278eaf072e9a62c03aa63a8a6bb branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Mariatta <Mariatta(a)users.noreply.github.com> date: 2018-01-29T20:16:21-08:00 summary: bpo-32701: Clarify the quotetabs flag in quopri documentation (GH-5401) (GH-5438) (cherry picked from commit 9424dcbb3e20a26dfdd81659303b989f7d3da044) Co-authored-by: Julien Palard <julien(a)palard.fr> files: M Doc/library/quopri.rst diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst index a3f94a0ad066..86717c00c3c1 100644 --- a/Doc/library/quopri.rst +++ b/Doc/library/quopri.rst @@ -34,9 +34,10 @@ sending a graphics file. Encode the contents of the *input* file and write the resulting quoted-printable data to the *output* file. *input* and *output* must be - :term:`binary file objects <file object>`. *quotetabs*, a flag which controls - whether to encode embedded spaces and tabs must be provideda and when true it - encodes such embedded whitespace, and when false it leaves them unencoded. + :term:`binary file objects <file object>`. *quotetabs*, a + non-optional flag which controls whether to encode embedded spaces + and tabs; when true it encodes such embedded whitespace, and when + false it leaves them unencoded. Note that spaces and tabs appearing at the end of lines are always encoded, as per :rfc:`1521`. *header* is a flag which controls if spaces are encoded as underscores as per :rfc:`1522`.

1 0

bpo-32604: NULL-terminate kwlist in channel_drop_interpreter(). (gh-5437)
by Eric Snow Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/83e64c8a544028ae677af2a0bc268dbe1c… commit: 83e64c8a544028ae677af2a0bc268dbe1c11cc3a branch: master author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-01-29T21:04:15-07:00 summary: bpo-32604: NULL-terminate kwlist in channel_drop_interpreter(). (gh-5437) files: M Modules/_xxsubinterpretersmodule.c diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index ed79a13d4a40..d2b5f26fae1d 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1916,7 +1916,7 @@ static PyObject * channel_drop_interpreter(PyObject *self, PyObject *args, PyObject *kwds) { // Note that only the current interpreter is affected. - static char *kwlist[] = {"id", "send", "recv"}; + static char *kwlist[] = {"id", "send", "recv", NULL}; PyObject *id; int send = -1; int recv = -1;

1 0

bpo-32701: Clarify the quotetabs flag in quopri documentation (GH-5401)
by Mariatta Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/9424dcbb3e20a26dfdd81659303b989f7d… commit: 9424dcbb3e20a26dfdd81659303b989f7d3da044 branch: master author: Julien Palard <julien(a)palard.fr> committer: Mariatta <Mariatta(a)users.noreply.github.com> date: 2018-01-29T19:36:06-08:00 summary: bpo-32701: Clarify the quotetabs flag in quopri documentation (GH-5401) files: M Doc/library/quopri.rst diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst index a3f94a0ad066..86717c00c3c1 100644 --- a/Doc/library/quopri.rst +++ b/Doc/library/quopri.rst @@ -34,9 +34,10 @@ sending a graphics file. Encode the contents of the *input* file and write the resulting quoted-printable data to the *output* file. *input* and *output* must be - :term:`binary file objects <file object>`. *quotetabs*, a flag which controls - whether to encode embedded spaces and tabs must be provideda and when true it - encodes such embedded whitespace, and when false it leaves them unencoded. + :term:`binary file objects <file object>`. *quotetabs*, a + non-optional flag which controls whether to encode embedded spaces + and tabs; when true it encodes such embedded whitespace, and when + false it leaves them unencoded. Note that spaces and tabs appearing at the end of lines are always encoded, as per :rfc:`1521`. *header* is a flag which controls if spaces are encoded as underscores as per :rfc:`1522`.

1 0

replace dynamic import with 'exec' with importlib.import_module (#5433)
by Gregory P. Smith Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/77526f05fa788d6fb12f2121fe6b96c130… commit: 77526f05fa788d6fb12f2121fe6b96c130d9b717 branch: master author: Benjamin Peterson <benjamin(a)python.org> committer: Gregory P. Smith <greg(a)krypto.org> date: 2018-01-29T18:03:01-08:00 summary: replace dynamic import with 'exec' with importlib.import_module (#5433) files: M Lib/test/test_hashlib.py diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index e4e5280dc8ea..15fc22b02d77 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -9,6 +9,7 @@ import array from binascii import unhexlify import hashlib +import importlib import itertools import os import sys @@ -83,11 +84,11 @@ class HashLibTestCase(unittest.TestCase): def _conditional_import_module(self, module_name): """Import a module and return a reference to it or None on failure.""" try: - exec('import '+module_name) - except ImportError as error: + return importlib.import_module(module_name) + except ModuleNotFoundError as error: if self._warn_on_extension_import: warnings.warn('Did a C extension fail to compile? %s' % error) - return locals().get(module_name) + return None def __init__(self, *args, **kwargs): algorithms = set()

1 0

bpo-32604: Expose the subinterpreters C-API in a "private" stdlib module. (gh-1748)
by Eric Snow Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/7f8bfc9b9a8381ddb768421b5dd5cbd970… commit: 7f8bfc9b9a8381ddb768421b5dd5cbd970266190 branch: master author: Eric Snow <ericsnowcurrently(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-01-29T18:23:44-07:00 summary: bpo-32604: Expose the subinterpreters C-API in a "private" stdlib module. (gh-1748) The module is primarily intended for internal use in the test suite. Building the module under Windows will come in a follow-up PR. files: A Lib/test/test__xxsubinterpreters.py A Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst A Modules/_xxsubinterpretersmodule.c M Include/internal/pystate.h M Python/pystate.c M setup.py diff --git a/Include/internal/pystate.h b/Include/internal/pystate.h index 0b464bcb2e86..2b60b25c19b1 100644 --- a/Include/internal/pystate.h +++ b/Include/internal/pystate.h @@ -65,6 +65,79 @@ PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate( PyAPI_FUNC(void) _PyPathConfig_Clear(_PyPathConfig *config); +/* interpreter state */ + +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterState_LookUpID(PY_INT64_T); + + +/* cross-interpreter data */ + +struct _xid; + +// _PyCrossInterpreterData is similar to Py_buffer as an effectively +// opaque struct that holds data outside the object machinery. This +// is necessary to pass between interpreters in the same process. +typedef struct _xid { + // data is the cross-interpreter-safe derivation of a Python object + // (see _PyObject_GetCrossInterpreterData). It will be NULL if the + // new_object func (below) encodes the data. + void *data; + // obj is the Python object from which the data was derived. This + // is non-NULL only if the data remains bound to the object in some + // way, such that the object must be "released" (via a decref) when + // the data is released. In that case it is automatically + // incref'ed (to match the automatic decref when releaed). + PyObject *obj; + // interp is the ID of the owning interpreter of the original + // object. It corresponds to the active interpreter when + // _PyObject_GetCrossInterpreterData() was called. This should only + // be set by the cross-interpreter machinery. + // + // We use the ID rather than the PyInterpreterState to avoid issues + // with deleted interpreters. + int64_t interp; + // new_object is a function that returns a new object in the current + // interpreter given the data. The resulting object (a new + // reference) will be equivalent to the original object. This field + // is required. + PyObject *(*new_object)(struct _xid *); + // free is called when the data is released. If it is NULL then + // nothing will be done to free the data. For some types this is + // okay (e.g. bytes) and for those types this field should be set + // to NULL. However, for most the data was allocated just for + // cross-interpreter use, so it must be freed when + // _PyCrossInterpreterData_Release is called or the memory will + // leak. In that case, at the very least this field should be set + // to PyMem_RawFree (the default if not explicitly set to NULL). + // The call will happen with the original interpreter activated. + void (*free)(void *); +} _PyCrossInterpreterData; + +typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *); + +PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *); +PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *); +PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *); + +/* cross-interpreter data registry */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc); +PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *); + +struct _xidregitem; + +struct _xidregitem { + PyTypeObject *cls; + crossinterpdatafunc getdata; + struct _xidregitem *next; +}; + + /* Full Python runtime state */ typedef struct pyruntimestate { @@ -86,6 +159,11 @@ typedef struct pyruntimestate { using a Python int. */ int64_t next_id; } interpreters; + // XXX Remove this field once we have a tp_* slot. + struct _xidregistry { + PyThread_type_lock mutex; + struct _xidregitem *head; + } xidregistry; #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); diff --git a/Lib/test/test__xxsubinterpreters.py b/Lib/test/test__xxsubinterpreters.py new file mode 100644 index 000000000000..2b170443a3b6 --- /dev/null +++ b/Lib/test/test__xxsubinterpreters.py @@ -0,0 +1,1118 @@ +import contextlib +import os +import pickle +from textwrap import dedent, indent +import threading +import unittest + +from test import support +from test.support import script_helper + +interpreters = support.import_module('_xxsubinterpreters') + + +def _captured_script(script): + r, w = os.pipe() + indented = script.replace('\n', '\n ') + wrapped = dedent(f""" + import contextlib + with open({w}, 'w') as chan: + with contextlib.redirect_stdout(chan): + {indented} + """) + return wrapped, open(r) + + +def _run_output(interp, request, shared=None): + script, chan = _captured_script(request) + with chan: + interpreters.run_string(interp, script, shared) + return chan.read() + + +(a)contextlib.contextmanager +def _running(interp): + r, w = os.pipe() + def run(): + interpreters.run_string(interp, dedent(f""" + # wait for "signal" + with open({r}) as chan: + chan.read() + """)) + + t = threading.Thread(target=run) + t.start() + + yield + + with open(w, 'w') as chan: + chan.write('done') + t.join() + + +class IsShareableTests(unittest.TestCase): + + def test_default_shareables(self): + shareables = [ + # singletons + None, + # builtin objects + b'spam', + ] + for obj in shareables: + with self.subTest(obj): + self.assertTrue( + interpreters.is_shareable(obj)) + + def test_not_shareable(self): + class Cheese: + def __init__(self, name): + self.name = name + def __str__(self): + return self.name + + class SubBytes(bytes): + """A subclass of a shareable type.""" + + not_shareables = [ + # singletons + True, + False, + NotImplemented, + ..., + # builtin types and objects + type, + object, + object(), + Exception(), + 42, + 100.0, + 'spam', + # user-defined types and objects + Cheese, + Cheese('Wensleydale'), + SubBytes(b'spam'), + ] + for obj in not_shareables: + with self.subTest(obj): + self.assertFalse( + interpreters.is_shareable(obj)) + + +class TestBase(unittest.TestCase): + + def tearDown(self): + for id in interpreters.list_all(): + if id == 0: # main + continue + try: + interpreters.destroy(id) + except RuntimeError: + pass # already destroyed + + for cid in interpreters.channel_list_all(): + try: + interpreters.channel_destroy(cid) + except interpreters.ChannelNotFoundError: + pass # already destroyed + + +class ListAllTests(TestBase): + + def test_initial(self): + main = interpreters.get_main() + ids = interpreters.list_all() + self.assertEqual(ids, [main]) + + def test_after_creating(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + ids = interpreters.list_all() + self.assertEqual(ids, [main, first, second]) + + def test_after_destroying(self): + main = interpreters.get_main() + first = interpreters.create() + second = interpreters.create() + interpreters.destroy(first) + ids = interpreters.list_all() + self.assertEqual(ids, [main, second]) + + +class GetCurrentTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + cur = interpreters.get_current() + self.assertEqual(cur, main) + + def test_subinterpreter(self): + main = interpreters.get_main() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(_interpreters.get_current()) + """)) + cur = int(out.strip()) + _, expected = interpreters.list_all() + self.assertEqual(cur, expected) + self.assertNotEqual(cur, main) + + +class GetMainTests(TestBase): + + def test_from_main(self): + [expected] = interpreters.list_all() + main = interpreters.get_main() + self.assertEqual(main, expected) + + def test_from_subinterpreter(self): + [expected] = interpreters.list_all() + interp = interpreters.create() + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(_interpreters.get_main()) + """)) + main = int(out.strip()) + self.assertEqual(main, expected) + + +class IsRunningTests(TestBase): + + def test_main(self): + main = interpreters.get_main() + self.assertTrue(interpreters.is_running(main)) + + def test_subinterpreter(self): + interp = interpreters.create() + self.assertFalse(interpreters.is_running(interp)) + + with _running(interp): + self.assertTrue(interpreters.is_running(interp)) + self.assertFalse(interpreters.is_running(interp)) + + def test_from_subinterpreter(self): + interp = interpreters.create() + out = _run_output(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + if _interpreters.is_running({interp}): + print(True) + else: + print(False) + """)) + self.assertEqual(out.strip(), 'True') + + def test_already_destroyed(self): + interp = interpreters.create() + interpreters.destroy(interp) + with self.assertRaises(RuntimeError): + interpreters.is_running(interp) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.is_running(-1) + + +class CreateTests(TestBase): + + def test_in_main(self): + id = interpreters.create() + + self.assertIn(id, interpreters.list_all()) + + @unittest.skip('enable this test when working on pystate.c') + def test_unique_id(self): + seen = set() + for _ in range(100): + id = interpreters.create() + interpreters.destroy(id) + seen.add(id) + + self.assertEqual(len(seen), 100) + + def test_in_thread(self): + lock = threading.Lock() + id = None + def f(): + nonlocal id + id = interpreters.create() + lock.acquire() + lock.release() + + t = threading.Thread(target=f) + with lock: + t.start() + t.join() + self.assertIn(id, interpreters.list_all()) + + def test_in_subinterpreter(self): + main, = interpreters.list_all() + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + print(id) + """)) + id2 = int(out.strip()) + + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + + def test_in_threaded_subinterpreter(self): + main, = interpreters.list_all() + id1 = interpreters.create() + id2 = None + def f(): + nonlocal id2 + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + id = _interpreters.create() + print(id) + """)) + id2 = int(out.strip()) + + t = threading.Thread(target=f) + t.start() + t.join() + + self.assertEqual(set(interpreters.list_all()), {main, id1, id2}) + + def test_after_destroy_all(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + ids = [] + for _ in range(3): + id = interpreters.create() + ids.append(id) + # Now destroy them. + for id in ids: + interpreters.destroy(id) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {id}) + + def test_after_destroy_some(self): + before = set(interpreters.list_all()) + # Create 3 subinterpreters. + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + # Now destroy 2 of them. + interpreters.destroy(id1) + interpreters.destroy(id3) + # Finally, create another. + id = interpreters.create() + self.assertEqual(set(interpreters.list_all()), before | {id, id2}) + + +class DestroyTests(TestBase): + + def test_one(self): + id1 = interpreters.create() + id2 = interpreters.create() + id3 = interpreters.create() + self.assertIn(id2, interpreters.list_all()) + interpreters.destroy(id2) + self.assertNotIn(id2, interpreters.list_all()) + self.assertIn(id1, interpreters.list_all()) + self.assertIn(id3, interpreters.list_all()) + + def test_all(self): + before = set(interpreters.list_all()) + ids = set() + for _ in range(3): + id = interpreters.create() + ids.add(id) + self.assertEqual(set(interpreters.list_all()), before | ids) + for id in ids: + interpreters.destroy(id) + self.assertEqual(set(interpreters.list_all()), before) + + def test_main(self): + main, = interpreters.list_all() + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + def f(): + with self.assertRaises(RuntimeError): + interpreters.destroy(main) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_already_destroyed(self): + id = interpreters.create() + interpreters.destroy(id) + with self.assertRaises(RuntimeError): + interpreters.destroy(id) + + def test_does_not_exist(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(1_000_000) + + def test_bad_id(self): + with self.assertRaises(RuntimeError): + interpreters.destroy(-1) + + def test_from_current(self): + main, = interpreters.list_all() + id = interpreters.create() + script = dedent(""" + import _xxsubinterpreters as _interpreters + _interpreters.destroy({}) + """).format(id) + + with self.assertRaises(RuntimeError): + interpreters.run_string(id, script) + self.assertEqual(set(interpreters.list_all()), {main, id}) + + def test_from_sibling(self): + main, = interpreters.list_all() + id1 = interpreters.create() + id2 = interpreters.create() + script = dedent(""" + import _xxsubinterpreters as _interpreters + _interpreters.destroy({}) + """).format(id2) + interpreters.run_string(id1, script) + + self.assertEqual(set(interpreters.list_all()), {main, id1}) + + def test_from_other_thread(self): + id = interpreters.create() + def f(): + interpreters.destroy(id) + + t = threading.Thread(target=f) + t.start() + t.join() + + def test_still_running(self): + main, = interpreters.list_all() + interp = interpreters.create() + with _running(interp): + with self.assertRaises(RuntimeError): + interpreters.destroy(interp) + self.assertTrue(interpreters.is_running(interp)) + + +class RunStringTests(TestBase): + + SCRIPT = dedent(""" + with open('{}', 'w') as out: + out.write('{}') + """) + FILENAME = 'spam' + + def setUp(self): + super().setUp() + self.id = interpreters.create() + self._fs = None + + def tearDown(self): + if self._fs is not None: + self._fs.close() + super().tearDown() + + @property + def fs(self): + if self._fs is None: + self._fs = FSFixture(self) + return self._fs + + def test_success(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + interpreters.run_string(self.id, script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_in_thread(self): + script, file = _captured_script('print("it worked!", end="")') + with file: + def f(): + interpreters.run_string(self.id, script) + + t = threading.Thread(target=f) + t.start() + t.join() + out = file.read() + + self.assertEqual(out, 'it worked!') + + def test_create_thread(self): + script, file = _captured_script(""" + import threading + def f(): + print('it worked!', end='') + + t = threading.Thread(target=f) + t.start() + t.join() + """) + with file: + interpreters.run_string(self.id, script) + out = file.read() + + self.assertEqual(out, 'it worked!') + + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_fork(self): + import tempfile + with tempfile.NamedTemporaryFile('w+') as file: + file.write('') + file.flush() + + expected = 'spam spam spam spam spam' + script = dedent(f""" + # (inspired by Lib/test/test_fork.py) + import os + pid = os.fork() + if pid == 0: # child + with open('{file.name}', 'w') as out: + out.write('{expected}') + # Kill the unittest runner in the child process. + os._exit(1) + else: + SHORT_SLEEP = 0.1 + import time + for _ in range(10): + spid, status = os.waitpid(pid, os.WNOHANG) + if spid == pid: + break + time.sleep(SHORT_SLEEP) + assert(spid == pid) + """) + interpreters.run_string(self.id, script) + + file.seek(0) + content = file.read() + self.assertEqual(content, expected) + + def test_already_running(self): + with _running(self.id): + with self.assertRaises(RuntimeError): + interpreters.run_string(self.id, 'print("spam")') + + def test_does_not_exist(self): + id = 0 + while id in interpreters.list_all(): + id += 1 + with self.assertRaises(RuntimeError): + interpreters.run_string(id, 'print("spam")') + + def test_error_id(self): + with self.assertRaises(RuntimeError): + interpreters.run_string(-1, 'print("spam")') + + def test_bad_id(self): + with self.assertRaises(TypeError): + interpreters.run_string('spam', 'print("spam")') + + def test_bad_script(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, 10) + + def test_bytes_for_script(self): + with self.assertRaises(TypeError): + interpreters.run_string(self.id, b'print("spam")') + + @contextlib.contextmanager + def assert_run_failed(self, exctype, msg=None): + with self.assertRaises(interpreters.RunFailedError) as caught: + yield + if msg is None: + self.assertEqual(str(caught.exception).split(':')[0], + str(exctype)) + else: + self.assertEqual(str(caught.exception), + "{}: {}".format(exctype, msg)) + + def test_invalid_syntax(self): + with self.assert_run_failed(SyntaxError): + # missing close paren + interpreters.run_string(self.id, 'print("spam"') + + def test_failure(self): + with self.assert_run_failed(Exception, 'spam'): + interpreters.run_string(self.id, 'raise Exception("spam")') + + def test_SystemExit(self): + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, 'raise SystemExit(42)') + + def test_sys_exit(self): + with self.assert_run_failed(SystemExit): + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit() + """)) + + with self.assert_run_failed(SystemExit, '42'): + interpreters.run_string(self.id, dedent(""" + import sys + sys.exit(42) + """)) + + def test_with_shared(self): + r, w = os.pipe() + + shared = { + 'spam': b'ham', + 'eggs': b'-1', + 'cheddar': None, + } + script = dedent(f""" + eggs = int(eggs) + spam = 42 + result = spam + eggs + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['spam'], 42) + self.assertEqual(ns['eggs'], -1) + self.assertEqual(ns['result'], 41) + self.assertIsNone(ns['cheddar']) + + def test_shared_overwrites(self): + interpreters.run_string(self.id, dedent(""" + spam = 'eggs' + ns1 = dict(vars()) + del ns1['__builtins__'] + """)) + + shared = {'spam': b'ham'} + script = dedent(f""" + ns2 = dict(vars()) + del ns2['__builtins__'] + """) + interpreters.run_string(self.id, script, shared) + + r, w = os.pipe() + script = dedent(f""" + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['ns1']['spam'], 'eggs') + self.assertEqual(ns['ns2']['spam'], b'ham') + self.assertEqual(ns['spam'], b'ham') + + def test_shared_overwrites_default_vars(self): + r, w = os.pipe() + + shared = {'__name__': b'not __main__'} + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script, shared) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + self.assertEqual(ns['__name__'], b'not __main__') + + def test_main_reused(self): + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + spam = True + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + del ns, pickle, chan + """)) + with open(r, 'rb') as chan: + ns1 = pickle.load(chan) + + r, w = os.pipe() + interpreters.run_string(self.id, dedent(f""" + eggs = False + + ns = dict(vars()) + del ns['__builtins__'] + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """)) + with open(r, 'rb') as chan: + ns2 = pickle.load(chan) + + self.assertIn('spam', ns1) + self.assertNotIn('eggs', ns1) + self.assertIn('eggs', ns2) + self.assertIn('spam', ns2) + + def test_execution_namespace_is_main(self): + r, w = os.pipe() + + script = dedent(f""" + spam = 42 + + ns = dict(vars()) + ns['__builtins__'] = str(ns['__builtins__']) + import pickle + with open({w}, 'wb') as chan: + pickle.dump(ns, chan) + """) + interpreters.run_string(self.id, script) + with open(r, 'rb') as chan: + ns = pickle.load(chan) + + ns.pop('__builtins__') + ns.pop('__loader__') + self.assertEqual(ns, { + '__name__': '__main__', + '__annotations__': {}, + '__doc__': None, + '__package__': None, + '__spec__': None, + 'spam': 42, + }) + + def test_still_running_at_exit(self): + script = dedent(f""" + from textwrap import dedent + import threading + import _xxsubinterpreters as _interpreters + def f(): + _interpreters.run_string(id, dedent(''' + import time + # Give plenty of time for the main interpreter to finish. + time.sleep(1_000_000) + ''')) + + t = threading.Thread(target=f) + t.start() + """) + with support.temp_dir() as dirname: + filename = script_helper.make_script(dirname, 'interp', script) + with script_helper.spawn_python(filename) as proc: + retcode = proc.wait() + + self.assertEqual(retcode, 0) + + +class ChannelIDTests(TestBase): + + def test_default_kwargs(self): + cid = interpreters._channel_id(10, force=True) + + self.assertEqual(int(cid), 10) + self.assertEqual(cid.end, 'both') + + def test_with_kwargs(self): + cid = interpreters._channel_id(10, send=True, force=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters._channel_id(10, send=True, recv=False, force=True) + self.assertEqual(cid.end, 'send') + + cid = interpreters._channel_id(10, recv=True, force=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters._channel_id(10, recv=True, send=False, force=True) + self.assertEqual(cid.end, 'recv') + + cid = interpreters._channel_id(10, send=True, recv=True, force=True) + self.assertEqual(cid.end, 'both') + + def test_coerce_id(self): + cid = interpreters._channel_id('10', force=True) + self.assertEqual(int(cid), 10) + + cid = interpreters._channel_id(10.0, force=True) + self.assertEqual(int(cid), 10) + + class Int(str): + def __init__(self, value): + self._value = value + def __int__(self): + return self._value + + cid = interpreters._channel_id(Int(10), force=True) + self.assertEqual(int(cid), 10) + + def test_bad_id(self): + ids = [-1, 2**64, "spam"] + for cid in ids: + with self.subTest(cid): + with self.assertRaises(ValueError): + interpreters._channel_id(cid) + + with self.assertRaises(TypeError): + interpreters._channel_id(object()) + + def test_bad_kwargs(self): + with self.assertRaises(ValueError): + interpreters._channel_id(10, send=False, recv=False) + + def test_does_not_exist(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters._channel_id(int(cid) + 1) # unforced + + def test_repr(self): + cid = interpreters._channel_id(10, force=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + cid = interpreters._channel_id(10, send=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10, send=True)') + + cid = interpreters._channel_id(10, recv=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10, recv=True)') + + cid = interpreters._channel_id(10, send=True, recv=True, force=True) + self.assertEqual(repr(cid), 'ChannelID(10)') + + def test_equality(self): + cid1 = interpreters.channel_create() + cid2 = interpreters._channel_id(int(cid1)) + cid3 = interpreters.channel_create() + + self.assertTrue(cid1 == cid1) + self.assertTrue(cid1 == cid2) + self.assertTrue(cid1 == int(cid1)) + self.assertFalse(cid1 == cid3) + + self.assertFalse(cid1 != cid1) + self.assertFalse(cid1 != cid2) + self.assertTrue(cid1 != cid3) + + +class ChannelTests(TestBase): + + def test_sequential_ids(self): + before = interpreters.channel_list_all() + id1 = interpreters.channel_create() + id2 = interpreters.channel_create() + id3 = interpreters.channel_create() + after = interpreters.channel_list_all() + + self.assertEqual(id2, int(id1) + 1) + self.assertEqual(id3, int(id2) + 1) + self.assertEqual(set(after) - set(before), {id1, id2, id3}) + + def test_ids_global(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + print(int(cid)) + """)) + cid1 = int(out.strip()) + + id2 = interpreters.create() + out = _run_output(id2, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + print(int(cid)) + """)) + cid2 = int(out.strip()) + + self.assertEqual(cid2, int(cid1) + 1) + + #################### + + def test_drop_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({int(cid)}, b'spam') + """)) + out = _run_output(id2, dedent(f""" + import _xxsubinterpreters as _interpreters + obj = _interpreters.channel_recv({int(cid)}) + _interpreters.channel_drop_interpreter({int(cid)}) + print(repr(obj)) + """)) + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + + self.assertEqual(out.strip(), "b'spam'") + + def test_drop_no_kwargs(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + def test_drop_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + obj = interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + self.assertEqual(obj, b'spam') + + def test_drop_close_if_unassociated(self): + cid = interpreters.channel_create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + obj = _interpreters.channel_send({int(cid)}, b'spam') + _interpreters.channel_drop_interpreter({int(cid)}) + """)) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_drop_partially(self): + # XXX Is partial close too wierd/confusing? + cid = interpreters.channel_create() + interpreters.channel_send(cid, None) + interpreters.channel_recv(cid) + interpreters.channel_send(cid, b'spam') + interpreters.channel_drop_interpreter(cid, send=True) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_drop_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_drop_interpreter(cid, send=True, recv=True) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + #################### + + def test_close_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_multiple_users(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + id2 = interpreters.create() + interpreters.run_string(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({int(cid)}, b'spam') + """)) + interpreters.run_string(id2, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_recv({int(cid)}) + """)) + interpreters.channel_close(cid) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id1, dedent(f""" + _interpreters.channel_send({int(cid)}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + with self.assertRaises(interpreters.RunFailedError) as cm: + interpreters.run_string(id2, dedent(f""" + _interpreters.channel_send({int(cid)}, b'spam') + """)) + self.assertIn('ChannelClosedError', str(cm.exception)) + + def test_close_multiple_times(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) + + def test_close_with_unused_items(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'ham') + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_never_used(self): + cid = interpreters.channel_create() + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'spam') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + def test_close_by_unassociated_interp(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_close({int(cid)}) + """)) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_close(cid) + + def test_close_used_multiple_times_by_single_user(self): + cid = interpreters.channel_create() + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_send(cid, b'spam') + interpreters.channel_recv(cid) + interpreters.channel_close(cid) + + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_send(cid, b'eggs') + with self.assertRaises(interpreters.ChannelClosedError): + interpreters.channel_recv(cid) + + #################### + + def test_send_recv_main(self): + cid = interpreters.channel_create() + orig = b'spam' + interpreters.channel_send(cid, orig) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, orig) + self.assertIsNot(obj, orig) + + def test_send_recv_same_interpreter(self): + id1 = interpreters.create() + out = _run_output(id1, dedent(""" + import _xxsubinterpreters as _interpreters + cid = _interpreters.channel_create() + orig = b'spam' + _interpreters.channel_send(cid, orig) + obj = _interpreters.channel_recv(cid) + assert obj is not orig + assert obj == orig + """)) + + def test_send_recv_different_interpreters(self): + cid = interpreters.channel_create() + id1 = interpreters.create() + out = _run_output(id1, dedent(f""" + import _xxsubinterpreters as _interpreters + _interpreters.channel_send({int(cid)}, b'spam') + """)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + + def test_send_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_send(10, b'spam') + + def test_recv_not_found(self): + with self.assertRaises(interpreters.ChannelNotFoundError): + interpreters.channel_recv(10) + + def test_recv_empty(self): + cid = interpreters.channel_create() + with self.assertRaises(interpreters.ChannelEmptyError): + interpreters.channel_recv(cid) + + def test_run_string_arg(self): + cid = interpreters.channel_create() + interp = interpreters.create() + + out = _run_output(interp, dedent(""" + import _xxsubinterpreters as _interpreters + print(cid.end) + _interpreters.channel_send(cid, b'spam') + """), + dict(cid=cid.send)) + obj = interpreters.channel_recv(cid) + + self.assertEqual(obj, b'spam') + self.assertEqual(out.strip(), 'send') + + +if __name__ == '__main__': + unittest.main() diff --git a/Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst b/Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst new file mode 100644 index 000000000000..f5472f9fe23f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2018-01-26-21-29-09.bpo-32604.7iazNx.rst @@ -0,0 +1,4 @@ +Add a new "_xxsubinterpreters" extension module that exposes the existing +subinterpreter C-API and a new cross-interpreter data sharing mechanism. The +module is primarily intended for more thorough testing of the existing +subinterpreter support. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c new file mode 100644 index 000000000000..ed79a13d4a40 --- /dev/null +++ b/Modules/_xxsubinterpretersmodule.c @@ -0,0 +1,2061 @@ + +/* interpreters module */ +/* low-level access to interpreter primitives */ + +#include "Python.h" +#include "frameobject.h" +#include "internal/pystate.h" + + +static PyInterpreterState * +_get_current(void) +{ + PyThreadState *tstate = PyThreadState_Get(); + // PyThreadState_Get() aborts if lookup fails, so we don't need + // to check the result for NULL. + return tstate->interp; +} + +static int64_t +_coerce_id(PyObject *id) +{ + id = PyNumber_Long(id); + if (id == NULL) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_SetString(PyExc_TypeError, + "'id' must be a non-negative int"); + } + else { + PyErr_SetString(PyExc_ValueError, + "'id' must be a non-negative int"); + } + return -1; + } + long long cid = PyLong_AsLongLong(id); + if (cid == -1 && PyErr_Occurred() != NULL) { + PyErr_SetString(PyExc_ValueError, + "'id' must be a non-negative int"); + return -1; + } + if (cid < 0) { + PyErr_SetString(PyExc_ValueError, + "'id' must be a non-negative int"); + return -1; + } + if (cid > INT64_MAX) { + PyErr_SetString(PyExc_ValueError, + "'id' too large (must be 64-bit int)"); + return -1; + } + return cid; +} + +/* data-sharing-specific code ***********************************************/ + +typedef struct _shareditem { + Py_UNICODE *name; + Py_ssize_t namelen; + _PyCrossInterpreterData data; +} _shareditem; + +void +_sharedns_clear(_shareditem *shared) +{ + for (_shareditem *item=shared; item->name != NULL; item += 1) { + _PyCrossInterpreterData_Release(&item->data); + } +} + +static _shareditem * +_get_shared_ns(PyObject *shareable, Py_ssize_t *lenp) +{ + if (shareable == NULL || shareable == Py_None) { + *lenp = 0; + return NULL; + } + Py_ssize_t len = PyDict_Size(shareable); + *lenp = len; + if (len == 0) { + return NULL; + } + + _shareditem *shared = PyMem_NEW(_shareditem, len+1); + if (shared == NULL) { + return NULL; + } + for (Py_ssize_t i=0; i < len; i++) { + *(shared + i) = (_shareditem){0}; + } + Py_ssize_t pos = 0; + for (Py_ssize_t i=0; i < len; i++) { + PyObject *key, *value; + if (PyDict_Next(shareable, &pos, &key, &value) == 0) { + break; + } + _shareditem *item = shared + i; + + if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) { + break; + } + item->name = PyUnicode_AsUnicodeAndSize(key, &item->namelen); + if (item->name == NULL) { + _PyCrossInterpreterData_Release(&item->data); + break; + } + (item + 1)->name = NULL; // Mark the next one as the last. + } + if (PyErr_Occurred()) { + _sharedns_clear(shared); + PyMem_Free(shared); + return NULL; + } + return shared; +} + +static int +_shareditem_apply(_shareditem *item, PyObject *ns) +{ + PyObject *name = PyUnicode_FromUnicode(item->name, item->namelen); + if (name == NULL) { + return 1; + } + PyObject *value = _PyCrossInterpreterData_NewObject(&item->data); + if (value == NULL) { + Py_DECREF(name); + return 1; + } + int res = PyDict_SetItem(ns, name, value); + Py_DECREF(name); + Py_DECREF(value); + return res; +} + +// Ultimately we'd like to preserve enough information about the +// exception and traceback that we could re-constitute (or at least +// simulate, a la traceback.TracebackException), and even chain, a copy +// of the exception in the calling interpreter. + +typedef struct _sharedexception { + char *msg; +} _sharedexception; + +static _sharedexception * +_get_shared_exception(void) +{ + _sharedexception *err = PyMem_NEW(_sharedexception, 1); + if (err == NULL) { + return NULL; + } + PyObject *exc; + PyObject *value; + PyObject *tb; + PyErr_Fetch(&exc, &value, &tb); + PyObject *msg; + if (value == NULL) { + msg = PyUnicode_FromFormat("%S", exc); + } + else { + msg = PyUnicode_FromFormat("%S: %S", exc, value); + } + if (msg == NULL) { + err->msg = "unable to format exception"; + return err; + } + err->msg = (char *)PyUnicode_AsUTF8(msg); + if (err->msg == NULL) { + err->msg = "unable to encode exception"; + } + return err; +} + +static PyObject * RunFailedError; + +static int +interp_exceptions_init(PyObject *ns) +{ + // XXX Move the exceptions into per-module memory? + + // An uncaught exception came out of interp_run_string(). + RunFailedError = PyErr_NewException("_xxsubinterpreters.RunFailedError", + PyExc_RuntimeError, NULL); + if (RunFailedError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "RunFailedError", RunFailedError) != 0) { + return -1; + } + + return 0; +} + +static void +_apply_shared_exception(_sharedexception *exc) +{ + PyErr_SetString(RunFailedError, exc->msg); +} + +/* channel-specific code */ + +static PyObject *ChannelError; +static PyObject *ChannelNotFoundError; +static PyObject *ChannelClosedError; +static PyObject *ChannelEmptyError; + +static int +channel_exceptions_init(PyObject *ns) +{ + // XXX Move the exceptions into per-module memory? + + // A channel-related operation failed. + ChannelError = PyErr_NewException("_xxsubinterpreters.ChannelError", + PyExc_RuntimeError, NULL); + if (ChannelError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "ChannelError", ChannelError) != 0) { + return -1; + } + + // An operation tried to use a channel that doesn't exist. + ChannelNotFoundError = PyErr_NewException( + "_xxsubinterpreters.ChannelNotFoundError", ChannelError, NULL); + if (ChannelNotFoundError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "ChannelNotFoundError", ChannelNotFoundError) != 0) { + return -1; + } + + // An operation tried to use a closed channel. + ChannelClosedError = PyErr_NewException( + "_xxsubinterpreters.ChannelClosedError", ChannelError, NULL); + if (ChannelClosedError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "ChannelClosedError", ChannelClosedError) != 0) { + return -1; + } + + // An operation tried to pop from an empty channel. + ChannelEmptyError = PyErr_NewException( + "_xxsubinterpreters.ChannelEmptyError", ChannelError, NULL); + if (ChannelEmptyError == NULL) { + return -1; + } + if (PyDict_SetItemString(ns, "ChannelEmptyError", ChannelEmptyError) != 0) { + return -1; + } + + return 0; +} + +struct _channelend; + +typedef struct _channelend { + struct _channelend *next; + int64_t interp; + int open; +} _channelend; + +static _channelend * +_channelend_new(int64_t interp) +{ + _channelend *end = PyMem_NEW(_channelend, 1); + if (end == NULL) { + return NULL; + } + + end->next = NULL; + end->interp = interp; + + end->open = 1; + + return end; +} + +static void +_channelend_free_all(_channelend *end) { + while (end != NULL) { + _channelend *last = end; + end = end->next; + PyMem_Free(last); + } +} + +static _channelend * +_channelend_find(_channelend *first, int64_t interp, _channelend **pprev) +{ + _channelend *prev = NULL; + _channelend *end = first; + while (end != NULL) { + if (end->interp == interp) { + break; + } + prev = end; + end = end->next; + } + if (pprev != NULL) { + *pprev = prev; + } + return end; +} + +struct _channelitem; + +typedef struct _channelitem { + _PyCrossInterpreterData *data; + struct _channelitem *next; +} _channelitem; + +struct _channel; + +typedef struct _channel { + PyThread_type_lock mutex; + + int open; + + int64_t count; + _channelitem *first; + _channelitem *last; + + // Note that the list entries are never removed for interpreter + // for which the channel is closed. This should be a problem in + // practice. Also, a channel isn't automatically closed when an + // interpreter is destroyed. + int64_t numsendopen; + int64_t numrecvopen; + _channelend *send; + _channelend *recv; +} _PyChannelState; + +static _PyChannelState * +_channel_new(void) +{ + _PyChannelState *chan = PyMem_NEW(_PyChannelState, 1); + if (chan == NULL) { + return NULL; + } + chan->mutex = PyThread_allocate_lock(); + if (chan->mutex == NULL) { + PyMem_Free(chan); + PyErr_SetString(ChannelError, + "can't initialize mutex for new channel"); + return NULL; + } + + chan->open = 1; + + chan->count = 0; + chan->first = NULL; + chan->last = NULL; + + chan->numsendopen = 0; + chan->numrecvopen = 0; + chan->send = NULL; + chan->recv = NULL; + + return chan; +} + +static _channelend * +_channel_add_end(_PyChannelState *chan, _channelend *prev, int64_t interp, + int send) +{ + _channelend *end = _channelend_new(interp); + if (end == NULL) { + return NULL; + } + + if (prev == NULL) { + if (send) { + chan->send = end; + } + else { + chan->recv = end; + } + } + else { + prev->next = end; + } + if (send) { + chan->numsendopen += 1; + } + else { + chan->numrecvopen += 1; + } + return end; +} + +static _channelend * +_channel_associate_end(_PyChannelState *chan, int64_t interp, int send) +{ + if (!chan->open) { + PyErr_SetString(ChannelClosedError, "channel closed"); + return NULL; + } + + _channelend *prev; + _channelend *end = _channelend_find(send ? chan->send : chan->recv, + interp, &prev); + if (end != NULL) { + if (!end->open) { + PyErr_SetString(ChannelClosedError, "channel already closed"); + return NULL; + } + // already associated + return end; + } + return _channel_add_end(chan, prev, interp, send); +} + +static void +_channel_close_channelend(_PyChannelState *chan, _channelend *end, int send) +{ + end->open = 0; + if (send) { + chan->numsendopen -= 1; + } + else { + chan->numrecvopen -= 1; + } +} + +static int +_channel_close_interpreter(_PyChannelState *chan, int64_t interp, int which) +{ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + + int res = -1; + if (!chan->open) { + PyErr_SetString(ChannelClosedError, "channel already closed"); + goto done; + } + + _channelend *prev; + _channelend *end; + if (which >= 0) { // send/both + end = _channelend_find(chan->send, interp, &prev); + if (end == NULL) { + // never associated so add it + end = _channel_add_end(chan, prev, interp, 1); + if (end == NULL) { + goto done; + } + } + _channel_close_channelend(chan, end, 1); + } + if (which <= 0) { // recv/both + end = _channelend_find(chan->recv, interp, &prev); + if (end == NULL) { + // never associated so add it + end = _channel_add_end(chan, prev, interp, 0); + if (end == NULL) { + goto done; + } + } + _channel_close_channelend(chan, end, 0); + } + + if (chan->numsendopen == 0 && chan->numrecvopen == 0) { + if (chan->send != NULL || chan->recv != NULL) { + chan->open = 0; + } + } + + res = 0; +done: + PyThread_release_lock(chan->mutex); + return res; +} + +static int +_channel_close_all(_PyChannelState *chan) +{ + int res = -1; + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + + if (!chan->open) { + PyErr_SetString(ChannelClosedError, "channel already closed"); + goto done; + } + + chan->open = 0; + + // We *could* also just leave these in place, since we've marked + // the channel as closed already. + + // Ensure all the "send"-associated interpreters are closed. + _channelend *end; + for (end = chan->send; end != NULL; end = end->next) { + _channel_close_channelend(chan, end, 1); + } + + // Ensure all the "recv"-associated interpreters are closed. + for (end = chan->recv; end != NULL; end = end->next) { + _channel_close_channelend(chan, end, 0); + } + + res = 0; +done: + PyThread_release_lock(chan->mutex); + return res; +} + +static int +_channel_add(_PyChannelState *chan, int64_t interp, + _PyCrossInterpreterData *data) +{ + int res = -1; + + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + if (_channel_associate_end(chan, interp, 1) == NULL) { + goto done; + } + + _channelitem *item = PyMem_NEW(_channelitem, 1); + if (item == NULL) { + goto done; + } + item->data = data; + item->next = NULL; + + chan->count += 1; + if (chan->first == NULL) { + chan->first = item; + } + chan->last = item; + + res = 0; +done: + PyThread_release_lock(chan->mutex); + return res; +} + +static _PyCrossInterpreterData * +_channel_next(_PyChannelState *chan, int64_t interp) +{ + _PyCrossInterpreterData *data = NULL; + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + if (_channel_associate_end(chan, interp, 0) == NULL) { + goto done; + } + + _channelitem *item = chan->first; + if (item == NULL) { + goto done; + } + chan->first = item->next; + if (chan->last == item) { + chan->last = NULL; + } + chan->count -= 1; + + data = item->data; + PyMem_Free(item); + +done: + PyThread_release_lock(chan->mutex); + return data; +} + +static void +_channel_clear(_PyChannelState *chan) +{ + _channelitem *item = chan->first; + while (item != NULL) { + _PyCrossInterpreterData_Release(item->data); + PyMem_Free(item->data); + _channelitem *last = item; + item = item->next; + PyMem_Free(last); + } + chan->first = NULL; + chan->last = NULL; +} + +static void +_channel_free(_PyChannelState *chan) +{ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + _channel_clear(chan); + _channelend_free_all(chan->send); + _channelend_free_all(chan->recv); + PyThread_release_lock(chan->mutex); + + PyThread_free_lock(chan->mutex); + PyMem_Free(chan); +} + +struct _channelref; + +typedef struct _channelref { + int64_t id; + _PyChannelState *chan; + struct _channelref *next; + Py_ssize_t objcount; +} _channelref; + +static _channelref * +_channelref_new(int64_t id, _PyChannelState *chan) +{ + _channelref *ref = PyMem_NEW(_channelref, 1); + if (ref == NULL) { + return NULL; + } + ref->id = id; + ref->chan = chan; + ref->next = NULL; + ref->objcount = 0; + return ref; +} + +static _channelref * +_channelref_find(_channelref *first, int64_t id, _channelref **pprev) +{ + _channelref *prev = NULL; + _channelref *ref = first; + while (ref != NULL) { + if (ref->id == id) { + break; + } + prev = ref; + ref = ref->next; + } + if (pprev != NULL) { + *pprev = prev; + } + return ref; +} + +typedef struct _channels { + PyThread_type_lock mutex; + _channelref *head; + int64_t numopen; + int64_t next_id; +} _channels; + +static int +_channels_init(_channels *channels) +{ + if (channels->mutex == NULL) { + channels->mutex = PyThread_allocate_lock(); + if (channels->mutex == NULL) { + PyMem_Free(channels); + PyErr_SetString(ChannelError, + "can't initialize mutex for channel management"); + return -1; + } + } + channels->head = NULL; + channels->numopen = 0; + channels->next_id = 0; + return 0; +} + +static int64_t +_channels_next_id(_channels *channels) // needs lock +{ + int64_t id = channels->next_id; + if (id < 0) { + /* overflow */ + PyErr_SetString(ChannelError, + "failed to get a channel ID"); + return -1; + } + channels->next_id += 1; + return id; +} + +static _PyChannelState * +_channels_lookup(_channels *channels, int64_t id, PyThread_type_lock *pmutex) +{ + _PyChannelState *chan = NULL; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + if (pmutex != NULL) { + *pmutex = NULL; + } + + _channelref *ref = _channelref_find(channels->head, id, NULL); + if (ref == NULL) { + PyErr_Format(ChannelNotFoundError, "channel %d not found", id); + goto done; + } + if (ref->chan == NULL || !ref->chan->open) { + PyErr_Format(ChannelClosedError, "channel %d closed", id); + goto done; + } + + if (pmutex != NULL) { + // The mutex will be closed by the caller. + *pmutex = channels->mutex; + } + + chan = ref->chan; +done: + if (pmutex == NULL || *pmutex == NULL) { + PyThread_release_lock(channels->mutex); + } + return chan; +} + +static int64_t +_channels_add(_channels *channels, _PyChannelState *chan) +{ + int64_t cid = -1; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + // Create a new ref. + int64_t id = _channels_next_id(channels); + if (id < 0) { + goto done; + } + _channelref *ref = _channelref_new(id, chan); + if (ref == NULL) { + goto done; + } + + // Add it to the list. + // We assume that the channel is a new one (not already in the list). + ref->next = channels->head; + channels->head = ref; + channels->numopen += 1; + + cid = id; +done: + PyThread_release_lock(channels->mutex); + return cid; +} + +static int +_channels_close(_channels *channels, int64_t cid, _PyChannelState **pchan) +{ + int res = -1; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + if (pchan != NULL) { + *pchan = NULL; + } + + _channelref *ref = _channelref_find(channels->head, cid, NULL); + if (ref == NULL) { + PyErr_Format(ChannelNotFoundError, "channel %d not found", cid); + goto done; + } + + if (ref->chan == NULL) { + PyErr_Format(ChannelClosedError, "channel %d closed", cid); + goto done; + } + else { + if (_channel_close_all(ref->chan) != 0) { + goto done; + } + if (pchan != NULL) { + *pchan = ref->chan; + } + ref->chan = NULL; + } + + res = 0; +done: + PyThread_release_lock(channels->mutex); + return res; +} + +static void +_channels_remove_ref(_channels *channels, _channelref *ref, _channelref *prev, + _PyChannelState **pchan) +{ + if (ref == channels->head) { + channels->head = ref->next; + } + else { + prev->next = ref->next; + } + channels->numopen -= 1; + + if (pchan != NULL) { + *pchan = ref->chan; + } + PyMem_Free(ref); +} + +static int +_channels_remove(_channels *channels, int64_t id, _PyChannelState **pchan) +{ + int res = -1; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + if (pchan != NULL) { + *pchan = NULL; + } + + _channelref *prev = NULL; + _channelref *ref = _channelref_find(channels->head, id, &prev); + if (ref == NULL) { + PyErr_Format(ChannelNotFoundError, "channel %d not found", id); + goto done; + } + + _channels_remove_ref(channels, ref, prev, pchan); + + res = 0; +done: + PyThread_release_lock(channels->mutex); + return res; +} + +static int +_channels_add_id_object(_channels *channels, int64_t id) +{ + int res = -1; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + _channelref *ref = _channelref_find(channels->head, id, NULL); + if (ref == NULL) { + PyErr_Format(ChannelNotFoundError, "channel %d not found", id); + goto done; + } + ref->objcount += 1; + + res = 0; +done: + PyThread_release_lock(channels->mutex); + return res; +} + +static void +_channels_drop_id_object(_channels *channels, int64_t id) +{ + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + _channelref *prev = NULL; + _channelref *ref = _channelref_find(channels->head, id, &prev); + if (ref == NULL) { + // Already destroyed. + goto done; + } + ref->objcount -= 1; + + // Destroy if no longer used. + if (ref->objcount == 0) { + _PyChannelState *chan = NULL; + _channels_remove_ref(channels, ref, prev, &chan); + if (chan != NULL) { + _channel_free(chan); + } + } + +done: + PyThread_release_lock(channels->mutex); +} + +int64_t * +_channels_list_all(_channels *channels, int64_t *count) +{ + int64_t *cids = NULL; + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + int64_t numopen = channels->numopen; + if (numopen >= PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_RuntimeError, "too many channels open"); + goto done; + } + int64_t *ids = PyMem_NEW(int64_t, (Py_ssize_t)(channels->numopen)); + if (ids == NULL) { + goto done; + } + _channelref *ref = channels->head; + for (int64_t i=0; ref != NULL; ref = ref->next, i++) { + ids[i] = ref->id; + } + *count = channels->numopen; + + cids = ids; +done: + PyThread_release_lock(channels->mutex); + return cids; +} + +/* "high"-level channel-related functions */ + +static int64_t +_channel_create(_channels *channels) +{ + _PyChannelState *chan = _channel_new(); + if (chan == NULL) { + return -1; + } + int64_t id = _channels_add(channels, chan); + if (id < 0) { + _channel_free(chan); + return -1; + } + return id; +} + +static int +_channel_destroy(_channels *channels, int64_t id) +{ + _PyChannelState *chan = NULL; + if (_channels_remove(channels, id, &chan) != 0) { + return -1; + } + if (chan != NULL) { + _channel_free(chan); + } + return 0; +} + +static int +_channel_send(_channels *channels, int64_t id, PyObject *obj) +{ + PyInterpreterState *interp = _get_current(); + if (interp == NULL) { + return -1; + } + + // Look up the channel. + PyThread_type_lock mutex = NULL; + _PyChannelState *chan = _channels_lookup(channels, id, &mutex); + if (chan == NULL) { + return -1; + } + // Past this point we are responsible for releasing the mutex. + + // Convert the object to cross-interpreter data. + _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); + if (data == NULL) { + PyThread_release_lock(mutex); + return -1; + } + if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { + PyThread_release_lock(mutex); + return -1; + } + + // Add the data to the channel. + int res = _channel_add(chan, interp->id, data); + PyThread_release_lock(mutex); + if (res != 0) { + _PyCrossInterpreterData_Release(data); + PyMem_Free(data); + return -1; + } + + return 0; +} + +static PyObject * +_channel_recv(_channels *channels, int64_t id) +{ + PyInterpreterState *interp = _get_current(); + if (interp == NULL) { + return NULL; + } + + // Look up the channel. + PyThread_type_lock mutex = NULL; + _PyChannelState *chan = _channels_lookup(channels, id, &mutex); + if (chan == NULL) { + return NULL; + } + // Past this point we are responsible for releasing the mutex. + + // Pop off the next item from the channel. + _PyCrossInterpreterData *data = _channel_next(chan, interp->id); + PyThread_release_lock(mutex); + if (data == NULL) { + PyErr_Format(ChannelEmptyError, "channel %d is empty", id); + return NULL; + } + + // Convert the data back to an object. + PyObject *obj = _PyCrossInterpreterData_NewObject(data); + if (obj == NULL) { + return NULL; + } + _PyCrossInterpreterData_Release(data); + + return obj; +} + +static int +_channel_drop(_channels *channels, int64_t id, int send, int recv) +{ + PyInterpreterState *interp = _get_current(); + if (interp == NULL) { + return -1; + } + + // Look up the channel. + PyThread_type_lock mutex = NULL; + _PyChannelState *chan = _channels_lookup(channels, id, &mutex); + if (chan == NULL) { + return -1; + } + // Past this point we are responsible for releasing the mutex. + + // Close one or both of the two ends. + int res =_channel_close_interpreter(chan, interp->id, send-recv); + PyThread_release_lock(mutex); + return res; +} + +static int +_channel_close(_channels *channels, int64_t id) +{ + return _channels_close(channels, id, NULL); +} + +/* ChannelID class */ + +#define CHANNEL_SEND 1 +#define CHANNEL_RECV -1 + +static PyTypeObject ChannelIDtype; + +typedef struct channelid { + PyObject_HEAD + int64_t id; + int end; + _channels *channels; +} channelid; + +static channelid * +newchannelid(PyTypeObject *cls, int64_t cid, int end, _channels *channels, + int force) +{ + channelid *self = PyObject_New(channelid, cls); + if (self == NULL) { + return NULL; + } + self->id = cid; + self->end = end; + self->channels = channels; + + if (_channels_add_id_object(channels, cid) != 0) { + if (force && PyErr_ExceptionMatches(ChannelNotFoundError)) { + PyErr_Clear(); + } + else { + Py_DECREF((PyObject *)self); + return NULL; + } + } + + return self; +} + +static _channels * _global_channels(void); + +static PyObject * +channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"id", "send", "recv", "force", NULL}; + PyObject *id; + int send = -1; + int recv = -1; + int force = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$ppp:ChannelID.__init__", kwlist, + &id, &send, &recv, &force)) + return NULL; + + // Coerce and check the ID. + int64_t cid; + if (PyObject_TypeCheck(id, &ChannelIDtype)) { + cid = ((channelid *)id)->id; + } + else { + cid = _coerce_id(id); + if (cid < 0) { + return NULL; + } + } + + // Handle "send" and "recv". + if (send == 0 && recv == 0) { + PyErr_SetString(PyExc_ValueError, + "'send' and 'recv' cannot both be False"); + return NULL; + } + int end = 0; + if (send == 1) { + if (recv == 0 || recv == -1) { + end = CHANNEL_SEND; + } + } + else if (recv == 1) { + end = CHANNEL_RECV; + } + + return (PyObject *)newchannelid(cls, cid, end, _global_channels(), force); +} + +static void +channelid_dealloc(PyObject *v) +{ + int64_t cid = ((channelid *)v)->id; + _channels *channels = ((channelid *)v)->channels; + Py_TYPE(v)->tp_free(v); + + _channels_drop_id_object(channels, cid); +} + +static PyObject * +channelid_repr(PyObject *self) +{ + PyTypeObject *type = Py_TYPE(self); + const char *name = _PyType_Name(type); + + channelid *cid = (channelid *)self; + const char *fmt; + if (cid->end == CHANNEL_SEND) { + fmt = "%s(%d, send=True)"; + } + else if (cid->end == CHANNEL_RECV) { + fmt = "%s(%d, recv=True)"; + } + else { + fmt = "%s(%d)"; + } + return PyUnicode_FromFormat(fmt, name, cid->id); +} + +PyObject * +channelid_int(PyObject *self) +{ + channelid *cid = (channelid *)self; + return PyLong_FromLongLong(cid->id); +} + +static PyNumberMethods channelid_as_number = { + 0, /* nb_add */ + 0, /* nb_subtract */ + 0, /* nb_multiply */ + 0, /* nb_remainder */ + 0, /* nb_divmod */ + 0, /* nb_power */ + 0, /* nb_negative */ + 0, /* nb_positive */ + 0, /* nb_absolute */ + 0, /* nb_bool */ + 0, /* nb_invert */ + 0, /* nb_lshift */ + 0, /* nb_rshift */ + 0, /* nb_and */ + 0, /* nb_xor */ + 0, /* nb_or */ + (unaryfunc)channelid_int, /* nb_int */ + 0, /* nb_reserved */ + 0, /* nb_float */ + + 0, /* nb_inplace_add */ + 0, /* nb_inplace_subtract */ + 0, /* nb_inplace_multiply */ + 0, /* nb_inplace_remainder */ + 0, /* nb_inplace_power */ + 0, /* nb_inplace_lshift */ + 0, /* nb_inplace_rshift */ + 0, /* nb_inplace_and */ + 0, /* nb_inplace_xor */ + 0, /* nb_inplace_or */ + + 0, /* nb_floor_divide */ + 0, /* nb_true_divide */ + 0, /* nb_inplace_floor_divide */ + 0, /* nb_inplace_true_divide */ + + (unaryfunc)channelid_int, /* nb_index */ +}; + +static Py_hash_t +channelid_hash(PyObject *self) +{ + channelid *cid = (channelid *)self; + PyObject *id = PyLong_FromLongLong(cid->id); + if (id == NULL) { + return -1; + } + return PyObject_Hash(id); +} + +static PyObject * +channelid_richcompare(PyObject *self, PyObject *other, int op) +{ + if (op != Py_EQ && op != Py_NE) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (!PyObject_TypeCheck(self, &ChannelIDtype)) { + Py_RETURN_NOTIMPLEMENTED; + } + + channelid *cid = (channelid *)self; + int equal; + if (PyObject_TypeCheck(other, &ChannelIDtype)) { + channelid *othercid = (channelid *)other; + if (cid->end != othercid->end) { + equal = 0; + } + else { + equal = (cid->id == othercid->id); + } + } + else { + other = PyNumber_Long(other); + if (other == NULL) { + PyErr_Clear(); + Py_RETURN_NOTIMPLEMENTED; + } + int64_t othercid = PyLong_AsLongLong(other); + // XXX decref other here? + if (othercid == -1 && PyErr_Occurred() != NULL) { + return NULL; + } + if (othercid < 0 || othercid > INT64_MAX) { + equal = 0; + } + else { + equal = (cid->id == othercid); + } + } + + if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +struct _channelid_xid { + int64_t id; + int end; +}; + +static PyObject * +_channelid_from_xid(_PyCrossInterpreterData *data) +{ + struct _channelid_xid *xid = (struct _channelid_xid *)data->data; + return (PyObject *)newchannelid(&ChannelIDtype, xid->id, xid->end, + _global_channels(), 0); +} + +static int +_channelid_shared(PyObject *obj, _PyCrossInterpreterData *data) +{ + struct _channelid_xid *xid = PyMem_NEW(struct _channelid_xid, 1); + if (xid == NULL) { + return -1; + } + xid->id = ((channelid *)obj)->id; + xid->end = ((channelid *)obj)->end; + + data->data = xid; + data->obj = obj; + data->new_object = _channelid_from_xid; + data->free = PyMem_Free; + return 0; +} + +static PyObject * +channelid_end(PyObject *self, void *end) +{ + int force = 1; + channelid *cid = (channelid *)self; + if (end != NULL) { + return (PyObject *)newchannelid(Py_TYPE(self), cid->id, *(int *)end, + cid->channels, force); + } + + if (cid->end == CHANNEL_SEND) { + return PyUnicode_InternFromString("send"); + } + if (cid->end == CHANNEL_RECV) { + return PyUnicode_InternFromString("recv"); + } + return PyUnicode_InternFromString("both"); +} + +static int _channelid_end_send = CHANNEL_SEND; +static int _channelid_end_recv = CHANNEL_RECV; + +static PyGetSetDef channelid_getsets[] = { + {"end", (getter)channelid_end, NULL, + PyDoc_STR("'send', 'recv', or 'both'")}, + {"send", (getter)channelid_end, NULL, + PyDoc_STR("the 'send' end of the channel"), &_channelid_end_send}, + {"recv", (getter)channelid_end, NULL, + PyDoc_STR("the 'recv' end of the channel"), &_channelid_end_recv}, + {NULL} +}; + +PyDoc_STRVAR(channelid_doc, +"A channel ID identifies a channel and may be used as an int."); + +static PyTypeObject ChannelIDtype = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "_xxsubinterpreters.ChannelID", /* tp_name */ + sizeof(channelid), /* tp_size */ + 0, /* tp_itemsize */ + (destructor)channelid_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_as_async */ + (reprfunc)channelid_repr, /* tp_repr */ + &channelid_as_number, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + channelid_hash, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_LONG_SUBCLASS, /* tp_flags */ + channelid_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + channelid_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + channelid_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + // Note that we do not set tp_new to channelid_new. Instead we + // set it to NULL, meaning it cannot be instantiated from Python + // code. We do this because there is a strong relationship between + // channel IDs and the channel lifecycle, so this limitation avoids + // related complications. + NULL, /* tp_new */ +}; + +/* interpreter-specific functions *******************************************/ + +static PyInterpreterState * +_look_up(PyObject *requested_id) +{ + long long id = PyLong_AsLongLong(requested_id); + if (id == -1 && PyErr_Occurred() != NULL) { + return NULL; + } + assert(id <= INT64_MAX); + return _PyInterpreterState_LookUpID(id); +} + +static PyObject * +_get_id(PyInterpreterState *interp) +{ + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) { + return NULL; + } + return PyLong_FromLongLong(id); +} + +static int +_is_running(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + if (PyThreadState_Next(tstate) != NULL) { + PyErr_SetString(PyExc_RuntimeError, + "interpreter has more than one thread"); + return -1; + } + PyFrameObject *frame = tstate->frame; + if (frame == NULL) { + if (PyErr_Occurred() != NULL) { + return -1; + } + return 0; + } + return (int)(frame->f_executing); +} + +static int +_ensure_not_running(PyInterpreterState *interp) +{ + int is_running = _is_running(interp); + if (is_running < 0) { + return -1; + } + if (is_running) { + PyErr_Format(PyExc_RuntimeError, "interpreter already running"); + return -1; + } + return 0; +} + +static int +_run_script(PyInterpreterState *interp, const char *codestr, + _shareditem *shared, Py_ssize_t num_shared, + _sharedexception **exc) +{ + assert(num_shared >= 0); + PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__"); + if (main_mod == NULL) { + goto error; + } + PyObject *ns = PyModule_GetDict(main_mod); // borrowed + Py_DECREF(main_mod); + if (ns == NULL) { + goto error; + } + Py_INCREF(ns); + + // Apply the cross-interpreter data. + if (shared != NULL) { + for (Py_ssize_t i=0; i < num_shared; i++) { + _shareditem *item = &shared[i]; + if (_shareditem_apply(item, ns) != 0) { + Py_DECREF(ns); + goto error; + } + } + } + + // Run the string (see PyRun_SimpleStringFlags). + PyObject *result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + Py_DECREF(ns); + if (result == NULL) { + goto error; + } + else { + Py_DECREF(result); // We throw away the result. + } + + return 0; + +error: + *exc = _get_shared_exception(); + PyErr_Clear(); + return -1; +} + +static int +_run_script_in_interpreter(PyInterpreterState *interp, const char *codestr, + PyObject *shareables) +{ + if (_ensure_not_running(interp) < 0) { + return -1; + } + + Py_ssize_t num_shared = -1; + _shareditem *shared = _get_shared_ns(shareables, &num_shared); + if (shared == NULL && PyErr_Occurred()) { + return -1; + } + + // Switch to interpreter. + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // Run the script. + _sharedexception *exc = NULL; + int result = _run_script(interp, codestr, shared, num_shared, &exc); + + // Switch back. + if (save_tstate != NULL) { + PyThreadState_Swap(save_tstate); + } + + // Propagate any exception out to the caller. + if (exc != NULL) { + _apply_shared_exception(exc); + PyMem_Free(exc); + } + else if (result != 0) { + // We were unable to allocate a shared exception. + PyErr_NoMemory(); + } + + if (shared != NULL) { + _sharedns_clear(shared); + PyMem_Free(shared); + } + + return result; +} + + +/* module level code ********************************************************/ + +/* globals is the process-global state for the module. It holds all + the data that we need to share between interpreters, so it cannot + hold PyObject values. */ +static struct globals { + _channels channels; +} _globals = {{0}}; + +static int +_init_globals(void) +{ + if (_channels_init(&_globals.channels) != 0) { + return -1; + } + return 0; +} + +static _channels * +_global_channels(void) { + return &_globals.channels; +} + +static PyObject * +interp_create(PyObject *self, PyObject *args) +{ + if (!PyArg_UnpackTuple(args, "create", 0, 0)) { + return NULL; + } + + // Create and initialize the new interpreter. + PyThreadState *tstate, *save_tstate; + save_tstate = PyThreadState_Swap(NULL); + tstate = Py_NewInterpreter(); + PyThreadState_Swap(save_tstate); + if (tstate == NULL) { + /* Since no new thread state was created, there is no exception to + propagate; raise a fresh one after swapping in the old thread + state. */ + PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed"); + return NULL; + } + return _get_id(tstate->interp); +} + +PyDoc_STRVAR(create_doc, +"create() -> ID\n\ +\n\ +Create a new interpreter and return a unique generated ID."); + + +static PyObject * +interp_destroy(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "destroy", 1, 1, &id)) { + return NULL; + } + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) { + return NULL; + } + + // Ensure we don't try to destroy the current interpreter. + PyInterpreterState *current = _get_current(); + if (current == NULL) { + return NULL; + } + if (interp == current) { + PyErr_SetString(PyExc_RuntimeError, + "cannot destroy the current interpreter"); + return NULL; + } + + // Ensure the interpreter isn't running. + /* XXX We *could* support destroying a running interpreter but + aren't going to worry about it for now. */ + if (_ensure_not_running(interp) < 0) { + return NULL; + } + + // Destroy the interpreter. + //PyInterpreterState_Delete(interp); + PyThreadState *tstate, *save_tstate; + tstate = PyInterpreterState_ThreadHead(interp); + save_tstate = PyThreadState_Swap(tstate); + Py_EndInterpreter(tstate); + PyThreadState_Swap(save_tstate); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(destroy_doc, +"destroy(ID)\n\ +\n\ +Destroy the identified interpreter.\n\ +\n\ +Attempting to destroy the current interpreter results in a RuntimeError.\n\ +So does an unrecognized ID."); + + +static PyObject * +interp_list_all(PyObject *self) +{ + PyObject *ids, *id; + PyInterpreterState *interp; + + ids = PyList_New(0); + if (ids == NULL) { + return NULL; + } + + interp = PyInterpreterState_Head(); + while (interp != NULL) { + id = _get_id(interp); + if (id == NULL) { + Py_DECREF(ids); + return NULL; + } + // insert at front of list + if (PyList_Insert(ids, 0, id) < 0) { + Py_DECREF(ids); + return NULL; + } + + interp = PyInterpreterState_Next(interp); + } + + return ids; +} + +PyDoc_STRVAR(list_all_doc, +"list_all() -> [ID]\n\ +\n\ +Return a list containing the ID of every existing interpreter."); + + +static PyObject * +interp_get_current(PyObject *self) +{ + PyInterpreterState *interp =_get_current(); + if (interp == NULL) { + return NULL; + } + return _get_id(interp); +} + +PyDoc_STRVAR(get_current_doc, +"get_current() -> ID\n\ +\n\ +Return the ID of current interpreter."); + + +static PyObject * +interp_get_main(PyObject *self) +{ + // Currently, 0 is always the main interpreter. + return PyLong_FromLongLong(0); +} + +PyDoc_STRVAR(get_main_doc, +"get_main() -> ID\n\ +\n\ +Return the ID of main interpreter."); + + +static PyObject * +interp_run_string(PyObject *self, PyObject *args) +{ + PyObject *id, *code; + PyObject *shared = NULL; + if (!PyArg_UnpackTuple(args, "run_string", 2, 3, &id, &code, &shared)) { + return NULL; + } + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "first arg (ID) must be an int"); + return NULL; + } + if (!PyUnicode_Check(code)) { + PyErr_SetString(PyExc_TypeError, + "second arg (code) must be a string"); + return NULL; + } + + // Look up the interpreter. + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) { + return NULL; + } + + // Extract code. + Py_ssize_t size; + const char *codestr = PyUnicode_AsUTF8AndSize(code, &size); + if (codestr == NULL) { + return NULL; + } + if (strlen(codestr) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + + // Run the code in the interpreter. + if (_run_script_in_interpreter(interp, codestr, shared) != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(run_string_doc, +"run_string(ID, sourcetext)\n\ +\n\ +Execute the provided string in the identified interpreter.\n\ +\n\ +See PyRun_SimpleStrings."); + + +static PyObject * +object_is_shareable(PyObject *self, PyObject *args) +{ + PyObject *obj; + if (!PyArg_UnpackTuple(args, "is_shareable", 1, 1, &obj)) { + return NULL; + } + if (_PyObject_CheckCrossInterpreterData(obj) == 0) { + Py_RETURN_TRUE; + } + PyErr_Clear(); + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_shareable_doc, +"is_shareable(obj) -> bool\n\ +\n\ +Return True if the object's data may be shared between interpreters and\n\ +False otherwise."); + + +static PyObject * +interp_is_running(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "is_running", 1, 1, &id)) { + return NULL; + } + if (!PyLong_Check(id)) { + PyErr_SetString(PyExc_TypeError, "ID must be an int"); + return NULL; + } + + PyInterpreterState *interp = _look_up(id); + if (interp == NULL) { + return NULL; + } + int is_running = _is_running(interp); + if (is_running < 0) { + return NULL; + } + if (is_running) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(is_running_doc, +"is_running(id) -> bool\n\ +\n\ +Return whether or not the identified interpreter is running."); + +static PyObject * +channel_create(PyObject *self) +{ + int64_t cid = _channel_create(&_globals.channels); + if (cid < 0) { + return NULL; + } + PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, cid, 0, + &_globals.channels, 0); + if (id == NULL) { + if (_channel_destroy(&_globals.channels, cid) != 0) { + // XXX issue a warning? + } + return NULL; + } + assert(((channelid *)id)->channels != NULL); + return id; +} + +PyDoc_STRVAR(channel_create_doc, +"channel_create() -> ID\n\ +\n\ +Create a new cross-interpreter channel and return a unique generated ID."); + +static PyObject * +channel_destroy(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "channel_destroy", 1, 1, &id)) { + return NULL; + } + int64_t cid = _coerce_id(id); + if (cid < 0) { + return NULL; + } + + if (_channel_destroy(&_globals.channels, cid) != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_destroy_doc, +"channel_destroy(ID)\n\ +\n\ +Close and finalize the channel. Afterward attempts to use the channel\n\ +will behave as though it never existed."); + +static PyObject * +channel_list_all(PyObject *self) +{ + int64_t count = 0; + int64_t *cids = _channels_list_all(&_globals.channels, &count); + if (cids == NULL) { + if (count == 0) { + return PyList_New(0); + } + return NULL; + } + PyObject *ids = PyList_New((Py_ssize_t)count); + if (ids == NULL) { + // XXX free cids + return NULL; + } + for (int64_t i=0; i < count; cids++, i++) { + PyObject *id = (PyObject *)newchannelid(&ChannelIDtype, *cids, 0, + &_globals.channels, 0); + if (id == NULL) { + Py_DECREF(ids); + ids = NULL; + break; + } + PyList_SET_ITEM(ids, i, id); + } + // XXX free cids + return ids; +} + +PyDoc_STRVAR(channel_list_all_doc, +"channel_list_all() -> [ID]\n\ +\n\ +Return the list of all IDs for active channels."); + +static PyObject * +channel_send(PyObject *self, PyObject *args) +{ + PyObject *id; + PyObject *obj; + if (!PyArg_UnpackTuple(args, "channel_send", 2, 2, &id, &obj)) { + return NULL; + } + int64_t cid = _coerce_id(id); + if (cid < 0) { + return NULL; + } + + if (_channel_send(&_globals.channels, cid, obj) != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_send_doc, +"channel_send(ID, obj)\n\ +\n\ +Add the object's data to the channel's queue."); + +static PyObject * +channel_recv(PyObject *self, PyObject *args) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) { + return NULL; + } + int64_t cid = _coerce_id(id); + if (cid < 0) { + return NULL; + } + + return _channel_recv(&_globals.channels, cid); +} + +PyDoc_STRVAR(channel_recv_doc, +"channel_recv(ID) -> obj\n\ +\n\ +Return a new object from the data at the from of the channel's queue."); + +static PyObject * +channel_close(PyObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *id; + if (!PyArg_UnpackTuple(args, "channel_recv", 1, 1, &id)) { + return NULL; + } + int64_t cid = _coerce_id(id); + if (cid < 0) { + return NULL; + } + + if (_channel_close(&_globals.channels, cid) != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_close_doc, +"channel_close(ID)\n\ +\n\ +Close the channel for all interpreters. Once the channel's ID has\n\ +no more ref counts the channel will be destroyed."); + +static PyObject * +channel_drop_interpreter(PyObject *self, PyObject *args, PyObject *kwds) +{ + // Note that only the current interpreter is affected. + static char *kwlist[] = {"id", "send", "recv"}; + PyObject *id; + int send = -1; + int recv = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|$pp:channel_drop_interpreter", kwlist, + &id, &send, &recv)) + return NULL; + + int64_t cid = _coerce_id(id); + if (cid < 0) { + return NULL; + } + if (send < 0 && recv < 0) { + send = 1; + recv = 1; + } + else { + if (send < 0) { + send = 0; + } + if (recv < 0) { + recv = 0; + } + } + if (_channel_drop(&_globals.channels, cid, send, recv) != 0) { + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(channel_drop_interpreter_doc, +"channel_drop_interpreter(ID, *, send=None, recv=None)\n\ +\n\ +Close the channel for the current interpreter. 'send' and 'recv'\n\ +(bool) may be used to indicate the ends to close. By default both\n\ +ends are closed. Closing an already closed end is a noop."); + +static PyObject * +channel__channel_id(PyObject *self, PyObject *args, PyObject *kwds) +{ + return channelid_new(&ChannelIDtype, args, kwds); +} + +static PyMethodDef module_functions[] = { + {"create", (PyCFunction)interp_create, + METH_VARARGS, create_doc}, + {"destroy", (PyCFunction)interp_destroy, + METH_VARARGS, destroy_doc}, + {"list_all", (PyCFunction)interp_list_all, + METH_NOARGS, list_all_doc}, + {"get_current", (PyCFunction)interp_get_current, + METH_NOARGS, get_current_doc}, + {"get_main", (PyCFunction)interp_get_main, + METH_NOARGS, get_main_doc}, + {"is_running", (PyCFunction)interp_is_running, + METH_VARARGS, is_running_doc}, + {"run_string", (PyCFunction)interp_run_string, + METH_VARARGS, run_string_doc}, + + {"is_shareable", (PyCFunction)object_is_shareable, + METH_VARARGS, is_shareable_doc}, + + {"channel_create", (PyCFunction)channel_create, + METH_NOARGS, channel_create_doc}, + {"channel_destroy", (PyCFunction)channel_destroy, + METH_VARARGS, channel_destroy_doc}, + {"channel_list_all", (PyCFunction)channel_list_all, + METH_NOARGS, channel_list_all_doc}, + {"channel_send", (PyCFunction)channel_send, + METH_VARARGS, channel_send_doc}, + {"channel_recv", (PyCFunction)channel_recv, + METH_VARARGS, channel_recv_doc}, + {"channel_close", (PyCFunction)channel_close, + METH_VARARGS, channel_close_doc}, + {"channel_drop_interpreter", (PyCFunction)channel_drop_interpreter, + METH_VARARGS | METH_KEYWORDS, channel_drop_interpreter_doc}, + {"_channel_id", (PyCFunction)channel__channel_id, + METH_VARARGS | METH_KEYWORDS, NULL}, + + {NULL, NULL} /* sentinel */ +}; + + +/* initialization function */ + +PyDoc_STRVAR(module_doc, +"This module provides primitive operations to manage Python interpreters.\n\ +The 'interpreters' module provides a more convenient interface."); + +static struct PyModuleDef interpretersmodule = { + PyModuleDef_HEAD_INIT, + "_xxsubinterpreters", /* m_name */ + module_doc, /* m_doc */ + -1, /* m_size */ + module_functions, /* m_methods */ + NULL, /* m_slots */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; + + +PyMODINIT_FUNC +PyInit__xxsubinterpreters(void) +{ + if (_init_globals() != 0) { + return NULL; + } + + /* Initialize types */ + ChannelIDtype.tp_base = &PyLong_Type; + if (PyType_Ready(&ChannelIDtype) != 0) { + return NULL; + } + + /* Create the module */ + PyObject *module = PyModule_Create(&interpretersmodule); + if (module == NULL) { + return NULL; + } + + /* Add exception types */ + PyObject *ns = PyModule_GetDict(module); // borrowed + if (interp_exceptions_init(ns) != 0) { + return NULL; + } + if (channel_exceptions_init(ns) != 0) { + return NULL; + } + + /* Add other types */ + Py_INCREF(&ChannelIDtype); + if (PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype) != 0) { + return NULL; + } + + if (_PyCrossInterpreterData_Register_Class(&ChannelIDtype, _channelid_shared)) { + return NULL; + } + + return module; +} diff --git a/Python/pystate.c b/Python/pystate.c index 909d831465d4..a474549a8c73 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -54,8 +54,13 @@ _PyRuntimeState_Init_impl(_PyRuntimeState *runtime) if (runtime->interpreters.mutex == NULL) { return _Py_INIT_ERR("Can't initialize threads for interpreter"); } - runtime->interpreters.next_id = -1; + + runtime->xidregistry.mutex = PyThread_allocate_lock(); + if (runtime->xidregistry.mutex == NULL) { + return _Py_INIT_ERR("Can't initialize threads for cross-interpreter data registry"); + } + return _Py_INIT_OK(); } @@ -166,6 +171,7 @@ PyInterpreterState_New(void) /* overflow or Py_Initialize() not called! */ PyErr_SetString(PyExc_RuntimeError, "failed to get an interpreter ID"); + /* XXX deallocate! */ interp = NULL; } else { interp->id = _PyRuntime.interpreters.next_id; @@ -256,6 +262,28 @@ PyInterpreterState_GetID(PyInterpreterState *interp) } +PyInterpreterState * +_PyInterpreterState_LookUpID(PY_INT64_T requested_id) +{ + if (requested_id < 0) + goto error; + + PyInterpreterState *interp = PyInterpreterState_Head(); + while (interp != NULL) { + PY_INT64_T id = PyInterpreterState_GetID(interp); + if (id < 0) + return NULL; + if (requested_id == id) + return interp; + interp = PyInterpreterState_Next(interp); + } + +error: + PyErr_Format(PyExc_RuntimeError, + "unrecognized interpreter ID %lld", requested_id); + return NULL; +} + /* Default implementation for _PyThreadState_GetFrame */ static struct _frame * threadstate_getframe(PyThreadState *self) @@ -1024,6 +1052,251 @@ PyGILState_Release(PyGILState_STATE oldstate) } +/**************************/ +/* cross-interpreter data */ +/**************************/ + +/* cross-interpreter data */ + +crossinterpdatafunc _PyCrossInterpreterData_Lookup(PyObject *); + +/* This is a separate func from _PyCrossInterpreterData_Lookup in order + to keep the registry code separate. */ +static crossinterpdatafunc +_lookup_getdata(PyObject *obj) +{ + crossinterpdatafunc getdata = _PyCrossInterpreterData_Lookup(obj); + if (getdata == NULL && PyErr_Occurred() == 0) + PyErr_Format(PyExc_ValueError, + "%S does not support cross-interpreter data", obj); + return getdata; +} + +int +_PyObject_CheckCrossInterpreterData(PyObject *obj) +{ + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + return -1; + } + return 0; +} + +static int +_check_xidata(_PyCrossInterpreterData *data) +{ + // data->data can be anything, including NULL, so we don't check it. + + // data->obj may be NULL, so we don't check it. + + if (data->interp < 0) { + PyErr_SetString(PyExc_SystemError, "missing interp"); + return -1; + } + + if (data->new_object == NULL) { + PyErr_SetString(PyExc_SystemError, "missing new_object func"); + return -1; + } + + // data->free may be NULL, so we don't check it. + + return 0; +} + +int +_PyObject_GetCrossInterpreterData(PyObject *obj, _PyCrossInterpreterData *data) +{ + PyThreadState *tstate = PyThreadState_Get(); + // PyThreadState_Get() aborts if lookup fails, so we don't need + // to check the result for NULL. + PyInterpreterState *interp = tstate->interp; + + // Reset data before re-populating. + *data = (_PyCrossInterpreterData){0}; + data->free = PyMem_RawFree; // Set a default that may be overridden. + + // Call the "getdata" func for the object. + Py_INCREF(obj); + crossinterpdatafunc getdata = _lookup_getdata(obj); + if (getdata == NULL) { + Py_DECREF(obj); + return -1; + } + int res = getdata(obj, data); + Py_DECREF(obj); + if (res != 0) { + return -1; + } + + // Fill in the blanks and validate the result. + Py_XINCREF(data->obj); + data->interp = interp->id; + if (_check_xidata(data) != 0) { + _PyCrossInterpreterData_Release(data); + return -1; + } + + return 0; +} + +void +_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data) +{ + if (data->data == NULL && data->obj == NULL) { + // Nothing to release! + return; + } + + // Switch to the original interpreter. + PyInterpreterState *interp = _PyInterpreterState_LookUpID(data->interp); + if (interp == NULL) { + // The intepreter was already destroyed. + if (data->free != NULL) { + // XXX Someone leaked some memory... + } + return; + } + PyThreadState *tstate = PyInterpreterState_ThreadHead(interp); + PyThreadState *save_tstate = PyThreadState_Swap(tstate); + + // "Release" the data and/or the object. + if (data->free != NULL) { + data->free(data->data); + } + Py_XDECREF(data->obj); + + // Switch back. + if (save_tstate != NULL) + PyThreadState_Swap(save_tstate); +} + +PyObject * +_PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data) +{ + return data->new_object(data); +} + +/* registry of {type -> crossinterpdatafunc} */ + +/* For now we use a global registry of shareable classes. An + alternative would be to add a tp_* slot for a class's + crossinterpdatafunc. It would be simpler and more efficient. */ + +static int +_register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata) +{ + // Note that we effectively replace already registered classes + // rather than failing. + struct _xidregitem *newhead = PyMem_RawMalloc(sizeof(struct _xidregitem)); + if (newhead == NULL) + return -1; + newhead->cls = cls; + newhead->getdata = getdata; + newhead->next = _PyRuntime.xidregistry.head; + _PyRuntime.xidregistry.head = newhead; + return 0; +} + +static void _register_builtins_for_crossinterpreter_data(void); + +int +_PyCrossInterpreterData_Register_Class(PyTypeObject *cls, + crossinterpdatafunc getdata) +{ + if (!PyType_Check(cls)) { + PyErr_Format(PyExc_ValueError, "only classes may be registered"); + return -1; + } + if (getdata == NULL) { + PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); + return -1; + } + + // Make sure the class isn't ever deallocated. + Py_INCREF((PyObject *)cls); + + PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK); + if (_PyRuntime.xidregistry.head == NULL) { + _register_builtins_for_crossinterpreter_data(); + } + int res = _register_xidata(cls, getdata); + PyThread_release_lock(_PyRuntime.xidregistry.mutex); + return res; +} + +crossinterpdatafunc +_PyCrossInterpreterData_Lookup(PyObject *obj) +{ + PyObject *cls = PyObject_Type(obj); + crossinterpdatafunc getdata = NULL; + PyThread_acquire_lock(_PyRuntime.xidregistry.mutex, WAIT_LOCK); + struct _xidregitem *cur = _PyRuntime.xidregistry.head; + if (cur == NULL) { + _register_builtins_for_crossinterpreter_data(); + cur = _PyRuntime.xidregistry.head; + } + for(; cur != NULL; cur = cur->next) { + if (cur->cls == (PyTypeObject *)cls) { + getdata = cur->getdata; + break; + } + } + PyThread_release_lock(_PyRuntime.xidregistry.mutex); + return getdata; +} + +/* cross-interpreter data for builtin types */ + +static PyObject * +_new_bytes_object(_PyCrossInterpreterData *data) +{ + return PyBytes_FromString((char *)(data->data)); +} + +static int +_bytes_shared(PyObject *obj, _PyCrossInterpreterData *data) +{ + data->data = (void *)(PyBytes_AS_STRING(obj)); + data->obj = obj; // Will be "released" (decref'ed) when data released. + data->new_object = _new_bytes_object; + data->free = NULL; // Do not free the data (it belongs to the object). + return 0; +} + +static PyObject * +_new_none_object(_PyCrossInterpreterData *data) +{ + // XXX Singleton refcounts are problematic across interpreters... + Py_INCREF(Py_None); + return Py_None; +} + +static int +_none_shared(PyObject *obj, _PyCrossInterpreterData *data) +{ + data->data = NULL; + // data->obj remains NULL + data->new_object = _new_none_object; + data->free = NULL; // There is nothing to free. + return 0; +} + +static void +_register_builtins_for_crossinterpreter_data(void) +{ + // None + if (_register_xidata((PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + Py_FatalError("could not register None for cross-interpreter sharing"); + } + + // bytes + if (_register_xidata(&PyBytes_Type, _bytes_shared) != 0) { + Py_FatalError("could not register bytes for cross-interpreter sharing"); + } +} + + #ifdef __cplusplus } #endif diff --git a/setup.py b/setup.py index ba0a7624cfcd..6f9a39719b43 100644 --- a/setup.py +++ b/setup.py @@ -755,6 +755,10 @@ def detect_modules(self): ['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c']) ) + # Python interface to subinterpreter C-API. + exts.append(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c'], + define_macros=[('Py_BUILD_CORE', '')])) + # # Here ends the simple stuff. From here on, modules need certain # libraries, are platform-specific, or present other surprises.

1 0

bpo-32550. Remove the STORE_ANNOTATION bytecode. (GH-5181)
by Raymond Hettinger Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/332cd5ee4ff42c9904c56e68a1028f383f… commit: 332cd5ee4ff42c9904c56e68a1028f383f7fc9a8 branch: master author: Mark Shannon <mark(a)hotpy.org> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2018-01-29T16:41:04-08:00 summary: bpo-32550. Remove the STORE_ANNOTATION bytecode. (GH-5181) files: A Misc/NEWS.d/next/Core and Builtins/2018-01-14-12-42-17.bpo-32550.k0EK-4.rst M Doc/library/dis.rst M Doc/whatsnew/3.7.rst M Include/opcode.h M Lib/importlib/_bootstrap_external.py M Lib/opcode.py M Lib/test/test_dis.py M Python/ceval.c M Python/compile.c M Python/importlib_external.h M Python/opcode_targets.h diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 01d46f88fb57..48c42d119d5d 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -993,13 +993,6 @@ All of the following opcodes use their arguments. Deletes local ``co_varnames[var_num]``. -.. opcode:: STORE_ANNOTATION (namei) - - Stores TOS as ``locals()['__annotations__'][co_names[namei]] = TOS``. - - .. versionadded:: 3.6 - - .. opcode:: LOAD_CLOSURE (i) Pushes a reference to the cell contained in slot *i* of the cell and free diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 71070df76470..4fdbb9182bbd 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -1173,6 +1173,8 @@ CPython bytecode changes * Added two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) +* Removed the STORE_ANNOTATION opcode. + Other CPython implementation changes ------------------------------------ diff --git a/Include/opcode.h b/Include/opcode.h index 99c3b0ef817a..fc6cbf3a7af4 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -99,7 +99,6 @@ extern "C" { #define LOAD_FAST 124 #define STORE_FAST 125 #define DELETE_FAST 126 -#define STORE_ANNOTATION 127 #define RAISE_VARARGS 130 #define CALL_FUNCTION 131 #define MAKE_FUNCTION 132 diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index cf75719c86f5..d3f58af504d8 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -242,6 +242,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.7a0 3390 (add LOAD_METHOD and CALL_METHOD opcodes) # Python 3.7a0 3391 (update GET_AITER #31709) # Python 3.7a0 3392 (PEP 552: Deterministic pycs) +# Python 3.7a0 3393 (remove STORE_ANNOTATION opcode) # # MAGIC must change whenever the bytecode emitted by the compiler may no # longer be understood by older implementations of the eval loop (usually @@ -250,7 +251,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3392).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3393).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' diff --git a/Lib/opcode.py b/Lib/opcode.py index dffb38c314a1..8f45f0a666f1 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -169,7 +169,6 @@ def jabs_op(name, op): haslocal.append(125) def_op('DELETE_FAST', 126) # Local variable number haslocal.append(126) -name_op('STORE_ANNOTATION', 127) # Index in name list def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) def_op('CALL_FUNCTION', 131) # #args diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 590f041aa926..ba8c6b940b04 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -226,23 +226,27 @@ def bug1333982(x=[]): 2 LOAD_CONST 0 (1) 4 STORE_NAME 0 (x) 6 LOAD_NAME 1 (int) - 8 STORE_ANNOTATION 0 (x) - - 3 10 LOAD_NAME 2 (fun) - 12 LOAD_CONST 0 (1) - 14 CALL_FUNCTION 1 - 16 STORE_ANNOTATION 3 (y) - - 4 18 LOAD_CONST 0 (1) - 20 LOAD_NAME 4 (lst) - 22 LOAD_NAME 2 (fun) - 24 LOAD_CONST 1 (0) - 26 CALL_FUNCTION 1 - 28 STORE_SUBSCR - 30 LOAD_NAME 1 (int) - 32 POP_TOP - 34 LOAD_CONST 2 (None) - 36 RETURN_VALUE + 8 LOAD_NAME 2 (__annotations__) + 10 LOAD_CONST 1 ('x') + 12 STORE_SUBSCR + + 3 14 LOAD_NAME 3 (fun) + 16 LOAD_CONST 0 (1) + 18 CALL_FUNCTION 1 + 20 LOAD_NAME 2 (__annotations__) + 22 LOAD_CONST 2 ('y') + 24 STORE_SUBSCR + + 4 26 LOAD_CONST 0 (1) + 28 LOAD_NAME 4 (lst) + 30 LOAD_NAME 3 (fun) + 32 LOAD_CONST 3 (0) + 34 CALL_FUNCTION 1 + 36 STORE_SUBSCR + 38 LOAD_NAME 1 (int) + 40 POP_TOP + 42 LOAD_CONST 4 (None) + 44 RETURN_VALUE """ compound_stmt_str = """\ diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-01-14-12-42-17.bpo-32550.k0EK-4.rst b/Misc/NEWS.d/next/Core and Builtins/2018-01-14-12-42-17.bpo-32550.k0EK-4.rst new file mode 100644 index 000000000000..9f77b94f6dd5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-01-14-12-42-17.bpo-32550.k0EK-4.rst @@ -0,0 +1 @@ +Remove the STORE_ANNOTATION bytecode. diff --git a/Python/ceval.c b/Python/ceval.c index 52a42b00724e..af5eb99d6c90 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1574,61 +1574,6 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) DISPATCH(); } - TARGET(STORE_ANNOTATION) { - _Py_IDENTIFIER(__annotations__); - PyObject *ann_dict; - PyObject *ann = POP(); - PyObject *name = GETITEM(names, oparg); - int err; - if (f->f_locals == NULL) { - PyErr_Format(PyExc_SystemError, - "no locals found when storing annotation"); - Py_DECREF(ann); - goto error; - } - /* first try to get __annotations__ from locals... */ - if (PyDict_CheckExact(f->f_locals)) { - ann_dict = _PyDict_GetItemId(f->f_locals, - &PyId___annotations__); - if (ann_dict == NULL) { - PyErr_SetString(PyExc_NameError, - "__annotations__ not found"); - Py_DECREF(ann); - goto error; - } - Py_INCREF(ann_dict); - } - else { - PyObject *ann_str = _PyUnicode_FromId(&PyId___annotations__); - if (ann_str == NULL) { - Py_DECREF(ann); - goto error; - } - ann_dict = PyObject_GetItem(f->f_locals, ann_str); - if (ann_dict == NULL) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_SetString(PyExc_NameError, - "__annotations__ not found"); - } - Py_DECREF(ann); - goto error; - } - } - /* ...if succeeded, __annotations__[name] = ann */ - if (PyDict_CheckExact(ann_dict)) { - err = PyDict_SetItem(ann_dict, name, ann); - } - else { - err = PyObject_SetItem(ann_dict, name, ann); - } - Py_DECREF(ann_dict); - Py_DECREF(ann); - if (err != 0) { - goto error; - } - DISPATCH(); - } - TARGET(DELETE_SUBSCR) { PyObject *sub = TOP(); PyObject *container = SECOND(); diff --git a/Python/compile.c b/Python/compile.c index 3e8323b933f4..0dc35662749b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -213,7 +213,7 @@ static int compiler_async_comprehension_generator( expr_ty elt, expr_ty val, int type); static PyCodeObject *assemble(struct compiler *, int addNone); -static PyObject *__doc__; +static PyObject *__doc__, *__annotations__; #define CAPSULE_NAME "compile.c compiler unit" @@ -311,7 +311,11 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, if (!__doc__) return NULL; } - + if (!__annotations__) { + __annotations__ = PyUnicode_InternFromString("__annotations__"); + if (!__annotations__) + return NULL; + } if (!compiler_init(&c)) return NULL; Py_INCREF(filename); @@ -1056,8 +1060,6 @@ stack_effect(int opcode, int oparg, int jump) return -1; case DELETE_FAST: return 0; - case STORE_ANNOTATION: - return -1; case RAISE_VARARGS: return -oparg; @@ -4711,8 +4713,10 @@ compiler_annassign(struct compiler *c, stmt_ty s) else { VISIT(c, expr, s->v.AnnAssign.annotation); } - /* ADDOP_N decrefs its argument */ - ADDOP_N(c, STORE_ANNOTATION, mangled, names); + ADDOP_NAME(c, LOAD_NAME, __annotations__, names); + ADDOP_O(c, LOAD_CONST, mangled, consts); + Py_DECREF(mangled); + ADDOP(c, STORE_SUBSCR); } break; case Attribute_kind: diff --git a/Python/importlib_external.h b/Python/importlib_external.h index e298bd0c20cd..6bf189d006ab 100644 --- a/Python/importlib_external.h +++ b/Python/importlib_external.h @@ -243,7 +243,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,4,0,0,0,218,13,95,119,114,105,116,101,95,97,116, 111,109,105,99,105,0,0,0,115,26,0,0,0,0,5,16, 1,6,1,26,1,2,3,14,1,20,1,16,1,14,1,2, - 1,14,1,14,1,6,1,114,56,0,0,0,105,64,13,0, + 1,14,1,14,1,6,1,114,56,0,0,0,105,65,13,0, 0,233,2,0,0,0,114,13,0,0,0,115,2,0,0,0, 13,10,90,11,95,95,112,121,99,97,99,104,101,95,95,122, 4,111,112,116,45,122,3,46,112,121,122,4,46,112,121,99, @@ -348,7 +348,7 @@ const unsigned char _Py_M__importlib_external[] = { 108,109,111,115,116,95,102,105,108,101,110,97,109,101,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,218,17,99, 97,99,104,101,95,102,114,111,109,95,115,111,117,114,99,101, - 9,1,0,0,115,48,0,0,0,0,18,8,1,6,1,6, + 10,1,0,0,115,48,0,0,0,0,18,8,1,6,1,6, 1,8,1,4,1,8,1,12,1,10,1,12,1,16,1,8, 1,8,1,8,1,24,1,8,1,12,1,6,2,8,1,8, 1,8,1,8,1,14,1,14,1,114,81,0,0,0,99,1, @@ -422,7 +422,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,110, 97,109,101,114,2,0,0,0,114,2,0,0,0,114,4,0, 0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,95, - 99,97,99,104,101,54,1,0,0,115,46,0,0,0,0,9, + 99,97,99,104,101,55,1,0,0,115,46,0,0,0,0,9, 12,1,8,1,10,1,12,1,12,1,8,1,6,1,10,1, 10,1,8,1,6,1,10,1,8,1,16,1,10,1,6,1, 8,1,16,1,8,1,6,1,8,1,14,1,114,87,0,0, @@ -456,7 +456,7 @@ const unsigned char _Py_M__importlib_external[] = { 36,0,0,0,90,9,101,120,116,101,110,115,105,111,110,218, 11,115,111,117,114,99,101,95,112,97,116,104,114,2,0,0, 0,114,2,0,0,0,114,4,0,0,0,218,15,95,103,101, - 116,95,115,111,117,114,99,101,102,105,108,101,88,1,0,0, + 116,95,115,111,117,114,99,101,102,105,108,101,89,1,0,0, 115,20,0,0,0,0,7,12,1,4,1,16,1,24,1,4, 1,2,1,12,1,18,1,18,1,114,93,0,0,0,99,1, 0,0,0,0,0,0,0,1,0,0,0,8,0,0,0,67, @@ -470,7 +470,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,68,0,0,0,114,76,0,0,0,41,1,218,8,102,105, 108,101,110,97,109,101,114,2,0,0,0,114,2,0,0,0, 114,4,0,0,0,218,11,95,103,101,116,95,99,97,99,104, - 101,100,107,1,0,0,115,16,0,0,0,0,1,14,1,2, + 101,100,108,1,0,0,115,16,0,0,0,0,1,14,1,2, 1,8,1,14,1,8,1,14,1,4,2,114,97,0,0,0, 99,1,0,0,0,0,0,0,0,2,0,0,0,8,0,0, 0,67,0,0,0,115,52,0,0,0,121,14,116,0,124,0, @@ -484,7 +484,7 @@ const unsigned char _Py_M__importlib_external[] = { 3,114,39,0,0,0,114,41,0,0,0,114,40,0,0,0, 41,2,114,35,0,0,0,114,42,0,0,0,114,2,0,0, 0,114,2,0,0,0,114,4,0,0,0,218,10,95,99,97, - 108,99,95,109,111,100,101,119,1,0,0,115,12,0,0,0, + 108,99,95,109,111,100,101,120,1,0,0,115,12,0,0,0, 0,2,2,1,14,1,14,1,10,3,8,1,114,99,0,0, 0,99,1,0,0,0,0,0,0,0,3,0,0,0,8,0, 0,0,3,0,0,0,115,68,0,0,0,100,6,135,0,102, @@ -521,7 +521,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,108,102,114,100,0,0,0,218,4,97,114,103,115,90,6, 107,119,97,114,103,115,41,1,218,6,109,101,116,104,111,100, 114,2,0,0,0,114,4,0,0,0,218,19,95,99,104,101, - 99,107,95,110,97,109,101,95,119,114,97,112,112,101,114,139, + 99,107,95,110,97,109,101,95,119,114,97,112,112,101,114,140, 1,0,0,115,12,0,0,0,0,1,8,1,8,1,10,1, 4,1,18,1,122,40,95,99,104,101,99,107,95,110,97,109, 101,46,60,108,111,99,97,108,115,62,46,95,99,104,101,99, @@ -539,7 +539,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,116,114,218,8,95,95,100,105,99,116,95,95,218,6,117, 112,100,97,116,101,41,3,90,3,110,101,119,90,3,111,108, 100,114,53,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,218,5,95,119,114,97,112,150,1,0,0, + 114,4,0,0,0,218,5,95,119,114,97,112,151,1,0,0, 115,8,0,0,0,0,1,10,1,10,1,22,1,122,26,95, 99,104,101,99,107,95,110,97,109,101,46,60,108,111,99,97, 108,115,62,46,95,119,114,97,112,41,1,78,41,3,218,10, @@ -547,7 +547,7 @@ const unsigned char _Py_M__importlib_external[] = { 9,78,97,109,101,69,114,114,111,114,41,3,114,104,0,0, 0,114,105,0,0,0,114,115,0,0,0,114,2,0,0,0, 41,1,114,104,0,0,0,114,4,0,0,0,218,11,95,99, - 104,101,99,107,95,110,97,109,101,131,1,0,0,115,14,0, + 104,101,99,107,95,110,97,109,101,132,1,0,0,115,14,0, 0,0,0,8,14,7,2,1,10,1,14,2,14,5,10,1, 114,118,0,0,0,99,2,0,0,0,0,0,0,0,5,0, 0,0,6,0,0,0,67,0,0,0,115,60,0,0,0,124, @@ -575,7 +575,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,218,6,108,111,97,100,101,114,218,8,112,111,114,116,105, 111,110,115,218,3,109,115,103,114,2,0,0,0,114,2,0, 0,0,114,4,0,0,0,218,17,95,102,105,110,100,95,109, - 111,100,117,108,101,95,115,104,105,109,159,1,0,0,115,10, + 111,100,117,108,101,95,115,104,105,109,160,1,0,0,115,10, 0,0,0,0,10,14,1,16,1,4,1,22,1,114,125,0, 0,0,99,3,0,0,0,0,0,0,0,6,0,0,0,4, 0,0,0,67,0,0,0,115,158,0,0,0,124,0,100,1, @@ -642,7 +642,7 @@ const unsigned char _Py_M__importlib_external[] = { 115,90,5,109,97,103,105,99,114,77,0,0,0,114,69,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, 0,218,13,95,99,108,97,115,115,105,102,121,95,112,121,99, - 176,1,0,0,115,28,0,0,0,0,16,12,1,8,1,16, + 177,1,0,0,115,28,0,0,0,0,16,12,1,8,1,16, 1,12,1,12,1,12,1,10,1,12,1,8,1,16,2,8, 1,16,1,12,1,114,133,0,0,0,99,5,0,0,0,0, 0,0,0,6,0,0,0,4,0,0,0,67,0,0,0,115, @@ -696,7 +696,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,114,100,0,0,0,114,132,0,0,0,114,77,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, 23,95,118,97,108,105,100,97,116,101,95,116,105,109,101,115, - 116,97,109,112,95,112,121,99,209,1,0,0,115,14,0,0, + 116,97,109,112,95,112,121,99,210,1,0,0,115,14,0,0, 0,0,19,24,1,10,1,12,1,12,1,8,1,24,1,114, 137,0,0,0,99,4,0,0,0,0,0,0,0,4,0,0, 0,3,0,0,0,67,0,0,0,115,38,0,0,0,124,0, @@ -742,7 +742,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,97,115,104,114,100,0,0,0,114,132,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,218,18,95, 118,97,108,105,100,97,116,101,95,104,97,115,104,95,112,121, - 99,237,1,0,0,115,8,0,0,0,0,17,16,1,2,1, + 99,238,1,0,0,115,8,0,0,0,0,17,16,1,2,1, 10,1,114,139,0,0,0,99,4,0,0,0,0,0,0,0, 5,0,0,0,5,0,0,0,67,0,0,0,115,80,0,0, 0,116,0,160,1,124,0,161,1,125,4,116,2,124,4,116, @@ -765,7 +765,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,100,0,0,0,114,91,0,0,0,114,92,0,0, 0,218,4,99,111,100,101,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,218,17,95,99,111,109,112,105,108,101, - 95,98,121,116,101,99,111,100,101,5,2,0,0,115,16,0, + 95,98,121,116,101,99,111,100,101,6,2,0,0,115,16,0, 0,0,0,2,10,1,10,1,12,1,8,1,12,1,4,2, 10,1,114,145,0,0,0,114,60,0,0,0,99,3,0,0, 0,0,0,0,0,4,0,0,0,5,0,0,0,67,0,0, @@ -783,7 +783,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,218,5,109,116,105,109,101,114,136,0,0,0,114,54, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, 0,0,218,22,95,99,111,100,101,95,116,111,95,116,105,109, - 101,115,116,97,109,112,95,112,121,99,18,2,0,0,115,12, + 101,115,116,97,109,112,95,112,121,99,19,2,0,0,115,12, 0,0,0,0,2,8,1,14,1,14,1,14,1,16,1,114, 150,0,0,0,84,99,3,0,0,0,0,0,0,0,5,0, 0,0,5,0,0,0,67,0,0,0,115,80,0,0,0,116, @@ -802,7 +802,7 @@ const unsigned char _Py_M__importlib_external[] = { 138,0,0,0,90,7,99,104,101,99,107,101,100,114,54,0, 0,0,114,69,0,0,0,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,218,17,95,99,111,100,101,95,116,111, - 95,104,97,115,104,95,112,121,99,28,2,0,0,115,14,0, + 95,104,97,115,104,95,112,121,99,29,2,0,0,115,14,0, 0,0,0,2,8,1,12,1,14,1,16,1,10,1,16,1, 114,152,0,0,0,99,1,0,0,0,0,0,0,0,5,0, 0,0,6,0,0,0,67,0,0,0,115,62,0,0,0,100, @@ -829,7 +829,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,108,105,110,101,218,8,101,110,99,111,100,105,110,103,90, 15,110,101,119,108,105,110,101,95,100,101,99,111,100,101,114, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, - 13,100,101,99,111,100,101,95,115,111,117,114,99,101,39,2, + 13,100,101,99,111,100,101,95,115,111,117,114,99,101,40,2, 0,0,115,10,0,0,0,0,5,8,1,12,1,10,1,12, 1,114,157,0,0,0,41,2,114,122,0,0,0,218,26,115, 117,98,109,111,100,117,108,101,95,115,101,97,114,99,104,95, @@ -891,7 +891,7 @@ const unsigned char _Py_M__importlib_external[] = { 115,114,161,0,0,0,90,7,100,105,114,110,97,109,101,114, 2,0,0,0,114,2,0,0,0,114,4,0,0,0,218,23, 115,112,101,99,95,102,114,111,109,95,102,105,108,101,95,108, - 111,99,97,116,105,111,110,56,2,0,0,115,62,0,0,0, + 111,99,97,116,105,111,110,57,2,0,0,115,62,0,0,0, 0,12,8,4,4,1,10,2,2,1,14,1,14,1,8,2, 10,8,16,1,6,3,8,1,16,1,14,1,10,1,6,1, 6,2,4,3,8,2,10,1,2,1,14,1,14,1,6,2, @@ -928,7 +928,7 @@ const unsigned char _Py_M__importlib_external[] = { 65,67,72,73,78,69,41,2,218,3,99,108,115,114,3,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, 0,218,14,95,111,112,101,110,95,114,101,103,105,115,116,114, - 121,136,2,0,0,115,8,0,0,0,0,2,2,1,14,1, + 121,137,2,0,0,115,8,0,0,0,0,2,2,1,14,1, 14,1,122,36,87,105,110,100,111,119,115,82,101,103,105,115, 116,114,121,70,105,110,100,101,114,46,95,111,112,101,110,95, 114,101,103,105,115,116,114,121,99,2,0,0,0,0,0,0, @@ -953,7 +953,7 @@ const unsigned char _Py_M__importlib_external[] = { 115,116,114,121,95,107,101,121,114,3,0,0,0,90,4,104, 107,101,121,218,8,102,105,108,101,112,97,116,104,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,218,16,95,115, - 101,97,114,99,104,95,114,101,103,105,115,116,114,121,143,2, + 101,97,114,99,104,95,114,101,103,105,115,116,114,121,144,2, 0,0,115,22,0,0,0,0,2,6,1,8,2,6,1,6, 1,22,1,2,1,12,1,26,1,14,1,6,1,122,38,87, 105,110,100,111,119,115,82,101,103,105,115,116,114,121,70,105, @@ -976,7 +976,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,116,114,178,0,0,0,114,122,0,0,0,114,168,0,0, 0,114,166,0,0,0,114,2,0,0,0,114,2,0,0,0, 114,4,0,0,0,218,9,102,105,110,100,95,115,112,101,99, - 158,2,0,0,115,26,0,0,0,0,2,10,1,8,1,4, + 159,2,0,0,115,26,0,0,0,0,2,10,1,8,1,4, 1,2,1,12,1,14,1,6,1,16,1,14,1,6,1,8, 1,8,1,122,31,87,105,110,100,111,119,115,82,101,103,105, 115,116,114,121,70,105,110,100,101,114,46,102,105,110,100,95, @@ -994,7 +994,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,41,2,114,182,0,0,0,114,122,0,0,0,41,4,114, 172,0,0,0,114,121,0,0,0,114,35,0,0,0,114,166, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, - 0,0,218,11,102,105,110,100,95,109,111,100,117,108,101,174, + 0,0,218,11,102,105,110,100,95,109,111,100,117,108,101,175, 2,0,0,115,8,0,0,0,0,7,12,1,8,1,6,2, 122,33,87,105,110,100,111,119,115,82,101,103,105,115,116,114, 121,70,105,110,100,101,114,46,102,105,110,100,95,109,111,100, @@ -1004,7 +1004,7 @@ const unsigned char _Py_M__importlib_external[] = { 11,99,108,97,115,115,109,101,116,104,111,100,114,173,0,0, 0,114,179,0,0,0,114,182,0,0,0,114,183,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,170,0,0,0,124,2,0,0,115,18,0, + 4,0,0,0,114,170,0,0,0,125,2,0,0,115,18,0, 0,0,12,5,4,3,4,2,4,2,12,7,12,15,2,1, 12,15,2,1,114,170,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,2,0,0,0,64,0,0,0,115,48, @@ -1039,7 +1039,7 @@ const unsigned char _Py_M__importlib_external[] = { 121,0,0,0,114,96,0,0,0,90,13,102,105,108,101,110, 97,109,101,95,98,97,115,101,90,9,116,97,105,108,95,110, 97,109,101,114,2,0,0,0,114,2,0,0,0,114,4,0, - 0,0,114,161,0,0,0,193,2,0,0,115,8,0,0,0, + 0,0,114,161,0,0,0,194,2,0,0,115,8,0,0,0, 0,3,18,1,16,1,14,1,122,24,95,76,111,97,100,101, 114,66,97,115,105,99,115,46,105,115,95,112,97,99,107,97, 103,101,99,2,0,0,0,0,0,0,0,2,0,0,0,1, @@ -1049,7 +1049,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,117,108,101,32,99,114,101,97,116,105,111,110,46,78,114, 2,0,0,0,41,2,114,102,0,0,0,114,166,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, - 13,99,114,101,97,116,101,95,109,111,100,117,108,101,201,2, + 13,99,114,101,97,116,101,95,109,111,100,117,108,101,202,2, 0,0,115,0,0,0,0,122,27,95,76,111,97,100,101,114, 66,97,115,105,99,115,46,99,114,101,97,116,101,95,109,111, 100,117,108,101,99,2,0,0,0,0,0,0,0,3,0,0, @@ -1069,7 +1069,7 @@ const unsigned char _Py_M__importlib_external[] = { 4,101,120,101,99,114,113,0,0,0,41,3,114,102,0,0, 0,218,6,109,111,100,117,108,101,114,144,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,218,11,101, - 120,101,99,95,109,111,100,117,108,101,204,2,0,0,115,10, + 120,101,99,95,109,111,100,117,108,101,205,2,0,0,115,10, 0,0,0,0,2,12,1,8,1,6,1,10,1,122,25,95, 76,111,97,100,101,114,66,97,115,105,99,115,46,101,120,101, 99,95,109,111,100,117,108,101,99,2,0,0,0,0,0,0, @@ -1080,14 +1080,14 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,218,17,95,108,111,97,100,95,109,111,100,117,108, 101,95,115,104,105,109,41,2,114,102,0,0,0,114,121,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,218,11,108,111,97,100,95,109,111,100,117,108,101,212,2, + 0,218,11,108,111,97,100,95,109,111,100,117,108,101,213,2, 0,0,115,2,0,0,0,0,2,122,25,95,76,111,97,100, 101,114,66,97,115,105,99,115,46,108,111,97,100,95,109,111, 100,117,108,101,78,41,8,114,107,0,0,0,114,106,0,0, 0,114,108,0,0,0,114,109,0,0,0,114,161,0,0,0, 114,187,0,0,0,114,192,0,0,0,114,194,0,0,0,114, 2,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,114,185,0,0,0,188,2,0,0,115,8,0,0, + 0,0,0,114,185,0,0,0,189,2,0,0,115,8,0,0, 0,12,5,8,8,8,3,8,8,114,185,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, 0,0,0,115,74,0,0,0,101,0,90,1,100,0,90,2, @@ -1112,7 +1112,7 @@ const unsigned char _Py_M__importlib_external[] = { 46,10,32,32,32,32,32,32,32,32,78,41,1,114,40,0, 0,0,41,2,114,102,0,0,0,114,35,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,218,10,112, - 97,116,104,95,109,116,105,109,101,219,2,0,0,115,2,0, + 97,116,104,95,109,116,105,109,101,220,2,0,0,115,2,0, 0,0,0,6,122,23,83,111,117,114,99,101,76,111,97,100, 101,114,46,112,97,116,104,95,109,116,105,109,101,99,2,0, 0,0,0,0,0,0,2,0,0,0,4,0,0,0,67,0, @@ -1147,7 +1147,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,32,114,149,0,0,0,41,1,114,196,0, 0,0,41,2,114,102,0,0,0,114,35,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,218,10,112, - 97,116,104,95,115,116,97,116,115,227,2,0,0,115,2,0, + 97,116,104,95,115,116,97,116,115,228,2,0,0,115,2,0, 0,0,0,11,122,23,83,111,117,114,99,101,76,111,97,100, 101,114,46,112,97,116,104,95,115,116,97,116,115,99,4,0, 0,0,0,0,0,0,4,0,0,0,4,0,0,0,67,0, @@ -1171,7 +1171,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,90,10,99,97,99,104,101,95,112,97,116,104,114, 54,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, 0,0,0,218,15,95,99,97,99,104,101,95,98,121,116,101, - 99,111,100,101,240,2,0,0,115,2,0,0,0,0,8,122, + 99,111,100,101,241,2,0,0,115,2,0,0,0,0,8,122, 28,83,111,117,114,99,101,76,111,97,100,101,114,46,95,99, 97,99,104,101,95,98,121,116,101,99,111,100,101,99,3,0, 0,0,0,0,0,0,3,0,0,0,1,0,0,0,67,0, @@ -1188,7 +1188,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,78,114,2,0,0,0,41,3,114,102,0, 0,0,114,35,0,0,0,114,54,0,0,0,114,2,0,0, 0,114,2,0,0,0,114,4,0,0,0,114,198,0,0,0, - 250,2,0,0,115,0,0,0,0,122,21,83,111,117,114,99, + 251,2,0,0,115,0,0,0,0,122,21,83,111,117,114,99, 101,76,111,97,100,101,114,46,115,101,116,95,100,97,116,97, 99,2,0,0,0,0,0,0,0,5,0,0,0,10,0,0, 0,67,0,0,0,115,82,0,0,0,124,0,160,0,124,1, @@ -1208,7 +1208,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,157,0,0,0,41,5,114,102,0,0,0,114,121,0,0, 0,114,35,0,0,0,114,155,0,0,0,218,3,101,120,99, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, - 10,103,101,116,95,115,111,117,114,99,101,1,3,0,0,115, + 10,103,101,116,95,115,111,117,114,99,101,2,3,0,0,115, 14,0,0,0,0,2,10,1,2,1,14,1,16,1,4,1, 28,1,122,23,83,111,117,114,99,101,76,111,97,100,101,114, 46,103,101,116,95,115,111,117,114,99,101,114,89,0,0,0, @@ -1230,7 +1230,7 @@ const unsigned char _Py_M__importlib_external[] = { 105,108,101,41,4,114,102,0,0,0,114,54,0,0,0,114, 35,0,0,0,114,203,0,0,0,114,2,0,0,0,114,2, 0,0,0,114,4,0,0,0,218,14,115,111,117,114,99,101, - 95,116,111,95,99,111,100,101,11,3,0,0,115,4,0,0, + 95,116,111,95,99,111,100,101,12,3,0,0,115,4,0,0, 0,0,5,12,1,122,27,83,111,117,114,99,101,76,111,97, 100,101,114,46,115,111,117,114,99,101,95,116,111,95,99,111, 100,101,99,2,0,0,0,0,0,0,0,15,0,0,0,9, @@ -1309,7 +1309,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,132,0,0,0,114,69,0,0,0,90,10,98,121, 116,101,115,95,100,97,116,97,90,11,99,111,100,101,95,111, 98,106,101,99,116,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,188,0,0,0,19,3,0,0,115,134,0, + 4,0,0,0,114,188,0,0,0,20,3,0,0,115,134,0, 0,0,0,7,10,1,4,1,4,1,4,1,4,1,4,1, 2,1,12,1,14,1,12,2,2,1,14,1,14,1,8,2, 12,1,2,1,14,1,14,1,6,3,2,1,8,2,2,1, @@ -1324,7 +1324,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,196,0,0,0,114,197,0,0,0,114,199,0,0,0, 114,198,0,0,0,114,202,0,0,0,114,206,0,0,0,114, 188,0,0,0,114,2,0,0,0,114,2,0,0,0,114,2, - 0,0,0,114,4,0,0,0,114,195,0,0,0,217,2,0, + 0,0,0,114,4,0,0,0,114,195,0,0,0,218,2,0, 0,115,14,0,0,0,8,2,8,8,8,13,8,10,8,7, 8,10,14,8,114,195,0,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,4,0,0,0,0,0,0,0,115,124, @@ -1354,7 +1354,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,2,114,100,0,0,0,114,35,0,0,0,41,3,114,102, 0,0,0,114,121,0,0,0,114,35,0,0,0,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,114,186,0,0, - 0,110,3,0,0,115,4,0,0,0,0,3,6,1,122,19, + 0,111,3,0,0,115,4,0,0,0,0,3,6,1,122,19, 70,105,108,101,76,111,97,100,101,114,46,95,95,105,110,105, 116,95,95,99,2,0,0,0,0,0,0,0,2,0,0,0, 2,0,0,0,67,0,0,0,115,24,0,0,0,124,0,106, @@ -1362,7 +1362,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,107,2,83,0,41,1,78,41,2,218,9,95,95,99,108, 97,115,115,95,95,114,113,0,0,0,41,2,114,102,0,0, 0,218,5,111,116,104,101,114,114,2,0,0,0,114,2,0, - 0,0,114,4,0,0,0,218,6,95,95,101,113,95,95,116, + 0,0,114,4,0,0,0,218,6,95,95,101,113,95,95,117, 3,0,0,115,4,0,0,0,0,1,12,1,122,17,70,105, 108,101,76,111,97,100,101,114,46,95,95,101,113,95,95,99, 1,0,0,0,0,0,0,0,1,0,0,0,3,0,0,0, @@ -1371,7 +1371,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,3,218,4,104,97,115,104,114,100,0,0,0,114,35,0, 0,0,41,1,114,102,0,0,0,114,2,0,0,0,114,2, 0,0,0,114,4,0,0,0,218,8,95,95,104,97,115,104, - 95,95,120,3,0,0,115,2,0,0,0,0,1,122,19,70, + 95,95,121,3,0,0,115,2,0,0,0,0,1,122,19,70, 105,108,101,76,111,97,100,101,114,46,95,95,104,97,115,104, 95,95,99,2,0,0,0,0,0,0,0,2,0,0,0,3, 0,0,0,3,0,0,0,115,16,0,0,0,116,0,116,1, @@ -1385,7 +1385,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,41,3,218,5,115,117,112,101,114,114,212,0, 0,0,114,194,0,0,0,41,2,114,102,0,0,0,114,121, 0,0,0,41,1,114,213,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,194,0,0,0,123,3,0,0,115,2,0, + 4,0,0,0,114,194,0,0,0,124,3,0,0,115,2,0, 0,0,0,10,122,22,70,105,108,101,76,111,97,100,101,114, 46,108,111,97,100,95,109,111,100,117,108,101,99,2,0,0, 0,0,0,0,0,2,0,0,0,1,0,0,0,67,0,0, @@ -1396,7 +1396,7 @@ const unsigned char _Py_M__importlib_external[] = { 104,101,32,102,105,110,100,101,114,46,41,1,114,35,0,0, 0,41,2,114,102,0,0,0,114,121,0,0,0,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,114,159,0,0, - 0,135,3,0,0,115,2,0,0,0,0,3,122,23,70,105, + 0,136,3,0,0,115,2,0,0,0,0,3,122,23,70,105, 108,101,76,111,97,100,101,114,46,103,101,116,95,102,105,108, 101,110,97,109,101,99,2,0,0,0,0,0,0,0,3,0, 0,0,9,0,0,0,67,0,0,0,115,32,0,0,0,116, @@ -1408,7 +1408,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,50,0,0,0,114,51,0,0,0,90,4,114,101,97,100, 41,3,114,102,0,0,0,114,35,0,0,0,114,55,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,200,0,0,0,140,3,0,0,115,4,0,0,0,0,2, + 114,200,0,0,0,141,3,0,0,115,4,0,0,0,0,2, 14,1,122,19,70,105,108,101,76,111,97,100,101,114,46,103, 101,116,95,100,97,116,97,99,2,0,0,0,0,0,0,0, 2,0,0,0,3,0,0,0,67,0,0,0,115,18,0,0, @@ -1416,7 +1416,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,83,0,41,1,78,41,1,114,161,0,0,0,41,2,114, 102,0,0,0,114,191,0,0,0,114,2,0,0,0,114,2, 0,0,0,114,4,0,0,0,218,19,103,101,116,95,114,101, - 115,111,117,114,99,101,95,114,101,97,100,101,114,147,3,0, + 115,111,117,114,99,101,95,114,101,97,100,101,114,148,3,0, 0,115,6,0,0,0,0,2,10,1,4,1,122,30,70,105, 108,101,76,111,97,100,101,114,46,103,101,116,95,114,101,115, 111,117,114,99,101,95,114,101,97,100,101,114,99,2,0,0, @@ -1429,7 +1429,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,102,0,0,0,218,8,114,101,115,111,117,114,99,101,114, 35,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, 0,0,0,218,13,111,112,101,110,95,114,101,115,111,117,114, - 99,101,153,3,0,0,115,4,0,0,0,0,1,20,1,122, + 99,101,154,3,0,0,115,4,0,0,0,0,1,20,1,122, 24,70,105,108,101,76,111,97,100,101,114,46,111,112,101,110, 95,114,101,115,111,117,114,99,101,99,2,0,0,0,0,0, 0,0,3,0,0,0,3,0,0,0,67,0,0,0,115,38, @@ -1442,7 +1442,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,41,3,114,102,0,0,0,114,221,0,0,0,114,35, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, 0,0,218,13,114,101,115,111,117,114,99,101,95,112,97,116, - 104,157,3,0,0,115,8,0,0,0,0,1,10,1,4,1, + 104,158,3,0,0,115,8,0,0,0,0,1,10,1,4,1, 20,1,122,24,70,105,108,101,76,111,97,100,101,114,46,114, 101,115,111,117,114,99,101,95,112,97,116,104,99,2,0,0, 0,0,0,0,0,3,0,0,0,3,0,0,0,67,0,0, @@ -1453,7 +1453,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,38,0,0,0,114,35,0,0,0,114,44,0, 0,0,41,3,114,102,0,0,0,114,100,0,0,0,114,35, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, - 0,0,114,223,0,0,0,163,3,0,0,115,8,0,0,0, + 0,0,114,223,0,0,0,164,3,0,0,115,8,0,0,0, 0,1,8,1,4,1,20,1,122,22,70,105,108,101,76,111, 97,100,101,114,46,105,115,95,114,101,115,111,117,114,99,101, 99,1,0,0,0,0,0,0,0,1,0,0,0,5,0,0, @@ -1463,7 +1463,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,114,114,1,0,0,0,218,7,108,105,115,116,100,105,114, 114,38,0,0,0,114,35,0,0,0,41,1,114,102,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 218,8,99,111,110,116,101,110,116,115,169,3,0,0,115,2, + 218,8,99,111,110,116,101,110,116,115,170,3,0,0,115,2, 0,0,0,0,1,122,19,70,105,108,101,76,111,97,100,101, 114,46,99,111,110,116,101,110,116,115,41,17,114,107,0,0, 0,114,106,0,0,0,114,108,0,0,0,114,109,0,0,0, @@ -1473,7 +1473,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,223,0,0,0,114,228,0,0,0,90,13,95,95, 99,108,97,115,115,99,101,108,108,95,95,114,2,0,0,0, 114,2,0,0,0,41,1,114,213,0,0,0,114,4,0,0, - 0,114,212,0,0,0,105,3,0,0,115,22,0,0,0,12, + 0,114,212,0,0,0,106,3,0,0,115,22,0,0,0,12, 5,8,6,8,4,8,3,16,12,12,5,8,7,12,6,8, 4,8,6,8,6,114,212,0,0,0,99,0,0,0,0,0, 0,0,0,0,0,0,0,3,0,0,0,64,0,0,0,115, @@ -1495,7 +1495,7 @@ const unsigned char _Py_M__importlib_external[] = { 115,116,95,109,116,105,109,101,90,7,115,116,95,115,105,122, 101,41,3,114,102,0,0,0,114,35,0,0,0,114,211,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,114,197,0,0,0,177,3,0,0,115,4,0,0,0,0, + 0,114,197,0,0,0,178,3,0,0,115,4,0,0,0,0, 2,8,1,122,27,83,111,117,114,99,101,70,105,108,101,76, 111,97,100,101,114,46,112,97,116,104,95,115,116,97,116,115, 99,4,0,0,0,0,0,0,0,5,0,0,0,5,0,0, @@ -1505,7 +1505,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,99,0,0,0,114,198,0,0,0,41,5,114,102,0,0, 0,114,92,0,0,0,114,91,0,0,0,114,54,0,0,0, 114,42,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,199,0,0,0,182,3,0,0,115,4,0, + 4,0,0,0,114,199,0,0,0,183,3,0,0,115,4,0, 0,0,0,2,8,1,122,32,83,111,117,114,99,101,70,105, 108,101,76,111,97,100,101,114,46,95,99,97,99,104,101,95, 98,121,116,101,99,111,100,101,105,182,1,0,0,41,1,114, @@ -1540,7 +1540,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,231,0,0,0,218,6,112,97,114,101,110,116,114,96,0, 0,0,114,27,0,0,0,114,23,0,0,0,114,201,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,198,0,0,0,187,3,0,0,115,42,0,0,0,0,2, + 114,198,0,0,0,188,3,0,0,115,42,0,0,0,0,2, 12,1,4,2,14,1,12,1,14,2,14,1,10,1,2,1, 14,1,14,2,6,1,16,3,6,1,8,1,22,1,2,1, 12,1,16,1,16,2,8,1,122,25,83,111,117,114,99,101, @@ -1549,7 +1549,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,108,0,0,0,114,109,0,0,0,114,197,0,0,0,114, 199,0,0,0,114,198,0,0,0,114,2,0,0,0,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,114,229,0, - 0,0,173,3,0,0,115,6,0,0,0,12,4,8,5,8, + 0,0,174,3,0,0,115,6,0,0,0,12,4,8,5,8, 5,114,229,0,0,0,99,0,0,0,0,0,0,0,0,0, 0,0,0,2,0,0,0,64,0,0,0,115,32,0,0,0, 101,0,90,1,100,0,90,2,100,1,90,3,100,2,100,3, @@ -1570,7 +1570,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,133,0,0,0,114,145,0,0,0,114,208,0,0,0, 41,5,114,102,0,0,0,114,121,0,0,0,114,35,0,0, 0,114,54,0,0,0,114,132,0,0,0,114,2,0,0,0, - 114,2,0,0,0,114,4,0,0,0,114,188,0,0,0,222, + 114,2,0,0,0,114,4,0,0,0,114,188,0,0,0,223, 3,0,0,115,18,0,0,0,0,1,10,1,10,4,2,1, 8,2,12,1,2,1,14,1,2,1,122,29,83,111,117,114, 99,101,108,101,115,115,70,105,108,101,76,111,97,100,101,114, @@ -1581,13 +1581,13 @@ const unsigned char _Py_M__importlib_external[] = { 105,115,32,110,111,32,115,111,117,114,99,101,32,99,111,100, 101,46,78,114,2,0,0,0,41,2,114,102,0,0,0,114, 121,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,114,202,0,0,0,238,3,0,0,115,2,0,0, + 0,0,0,114,202,0,0,0,239,3,0,0,115,2,0,0, 0,0,2,122,31,83,111,117,114,99,101,108,101,115,115,70, 105,108,101,76,111,97,100,101,114,46,103,101,116,95,115,111, 117,114,99,101,78,41,6,114,107,0,0,0,114,106,0,0, 0,114,108,0,0,0,114,109,0,0,0,114,188,0,0,0, 114,202,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 2,0,0,0,114,4,0,0,0,114,234,0,0,0,218,3, + 2,0,0,0,114,4,0,0,0,114,234,0,0,0,219,3, 0,0,115,4,0,0,0,12,4,8,16,114,234,0,0,0, 99,0,0,0,0,0,0,0,0,0,0,0,0,3,0,0, 0,64,0,0,0,115,92,0,0,0,101,0,90,1,100,0, @@ -1609,7 +1609,7 @@ const unsigned char _Py_M__importlib_external[] = { 95,1,100,0,83,0,41,1,78,41,2,114,100,0,0,0, 114,35,0,0,0,41,3,114,102,0,0,0,114,100,0,0, 0,114,35,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,114,186,0,0,0,255,3,0,0,115,4, + 114,4,0,0,0,114,186,0,0,0,0,4,0,0,115,4, 0,0,0,0,1,6,1,122,28,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,105, 110,105,116,95,95,99,2,0,0,0,0,0,0,0,2,0, @@ -1618,7 +1618,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,106,1,107,2,83,0,41,1,78,41,2,114,213,0,0, 0,114,113,0,0,0,41,2,114,102,0,0,0,114,214,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,114,215,0,0,0,3,4,0,0,115,4,0,0,0,0, + 0,114,215,0,0,0,4,4,0,0,115,4,0,0,0,0, 1,12,1,122,26,69,120,116,101,110,115,105,111,110,70,105, 108,101,76,111,97,100,101,114,46,95,95,101,113,95,95,99, 1,0,0,0,0,0,0,0,1,0,0,0,3,0,0,0, @@ -1626,7 +1626,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,116,0,124,0,106,2,131,1,65,0,83,0,41,1,78, 41,3,114,216,0,0,0,114,100,0,0,0,114,35,0,0, 0,41,1,114,102,0,0,0,114,2,0,0,0,114,2,0, - 0,0,114,4,0,0,0,114,217,0,0,0,7,4,0,0, + 0,0,114,4,0,0,0,114,217,0,0,0,8,4,0,0, 115,2,0,0,0,0,1,122,28,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,95,95,104, 97,115,104,95,95,99,2,0,0,0,0,0,0,0,3,0, @@ -1643,7 +1643,7 @@ const unsigned char _Py_M__importlib_external[] = { 121,110,97,109,105,99,114,130,0,0,0,114,100,0,0,0, 114,35,0,0,0,41,3,114,102,0,0,0,114,166,0,0, 0,114,191,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,114,187,0,0,0,10,4,0,0,115,10, + 114,4,0,0,0,114,187,0,0,0,11,4,0,0,115,10, 0,0,0,0,2,4,1,10,1,6,1,12,1,122,33,69, 120,116,101,110,115,105,111,110,70,105,108,101,76,111,97,100, 101,114,46,99,114,101,97,116,101,95,109,111,100,117,108,101, @@ -1660,7 +1660,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,99,95,100,121,110,97,109,105,99,114,130,0,0,0,114, 100,0,0,0,114,35,0,0,0,41,2,114,102,0,0,0, 114,191,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,192,0,0,0,18,4,0,0,115,6,0, + 4,0,0,0,114,192,0,0,0,19,4,0,0,115,6,0, 0,0,0,2,14,1,6,1,122,31,69,120,116,101,110,115, 105,111,110,70,105,108,101,76,111,97,100,101,114,46,101,120, 101,99,95,109,111,100,117,108,101,99,2,0,0,0,0,0, @@ -1678,7 +1678,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,41,2,114,22,0,0,0,218,6,115,117,102,102,105, 120,41,1,218,9,102,105,108,101,95,110,97,109,101,114,2, 0,0,0,114,4,0,0,0,250,9,60,103,101,110,101,120, - 112,114,62,27,4,0,0,115,2,0,0,0,4,1,122,49, + 112,114,62,28,4,0,0,115,2,0,0,0,4,1,122,49, 69,120,116,101,110,115,105,111,110,70,105,108,101,76,111,97, 100,101,114,46,105,115,95,112,97,99,107,97,103,101,46,60, 108,111,99,97,108,115,62,46,60,103,101,110,101,120,112,114, @@ -1686,7 +1686,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,121,218,18,69,88,84,69,78,83,73,79,78,95,83,85, 70,70,73,88,69,83,41,2,114,102,0,0,0,114,121,0, 0,0,114,2,0,0,0,41,1,114,237,0,0,0,114,4, - 0,0,0,114,161,0,0,0,24,4,0,0,115,6,0,0, + 0,0,0,114,161,0,0,0,25,4,0,0,115,6,0,0, 0,0,2,14,1,12,1,122,30,69,120,116,101,110,115,105, 111,110,70,105,108,101,76,111,97,100,101,114,46,105,115,95, 112,97,99,107,97,103,101,99,2,0,0,0,0,0,0,0, @@ -1697,7 +1697,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,116,32,99,114,101,97,116,101,32,97,32,99,111,100,101, 32,111,98,106,101,99,116,46,78,114,2,0,0,0,41,2, 114,102,0,0,0,114,121,0,0,0,114,2,0,0,0,114, - 2,0,0,0,114,4,0,0,0,114,188,0,0,0,30,4, + 2,0,0,0,114,4,0,0,0,114,188,0,0,0,31,4, 0,0,115,2,0,0,0,0,2,122,28,69,120,116,101,110, 115,105,111,110,70,105,108,101,76,111,97,100,101,114,46,103, 101,116,95,99,111,100,101,99,2,0,0,0,0,0,0,0, @@ -1708,7 +1708,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,32,115,111,117,114,99,101,32,99,111,100,101,46,78,114, 2,0,0,0,41,2,114,102,0,0,0,114,121,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,114, - 202,0,0,0,34,4,0,0,115,2,0,0,0,0,2,122, + 202,0,0,0,35,4,0,0,115,2,0,0,0,0,2,122, 30,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, 97,100,101,114,46,103,101,116,95,115,111,117,114,99,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, @@ -1719,7 +1719,7 @@ const unsigned char _Py_M__importlib_external[] = { 121,32,116,104,101,32,102,105,110,100,101,114,46,41,1,114, 35,0,0,0,41,2,114,102,0,0,0,114,121,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,114, - 159,0,0,0,38,4,0,0,115,2,0,0,0,0,3,122, + 159,0,0,0,39,4,0,0,115,2,0,0,0,0,3,122, 32,69,120,116,101,110,115,105,111,110,70,105,108,101,76,111, 97,100,101,114,46,103,101,116,95,102,105,108,101,110,97,109, 101,78,41,14,114,107,0,0,0,114,106,0,0,0,114,108, @@ -1728,7 +1728,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,161,0,0,0,114,188,0,0,0,114,202,0,0,0, 114,118,0,0,0,114,159,0,0,0,114,2,0,0,0,114, 2,0,0,0,114,2,0,0,0,114,4,0,0,0,114,235, - 0,0,0,247,3,0,0,115,18,0,0,0,12,8,8,4, + 0,0,0,248,3,0,0,115,18,0,0,0,12,8,8,4, 8,4,8,3,8,8,8,6,8,6,8,4,8,4,114,235, 0,0,0,99,0,0,0,0,0,0,0,0,0,0,0,0, 2,0,0,0,64,0,0,0,115,96,0,0,0,101,0,90, @@ -1769,7 +1769,7 @@ const unsigned char _Py_M__importlib_external[] = { 100,101,114,41,4,114,102,0,0,0,114,100,0,0,0,114, 35,0,0,0,218,11,112,97,116,104,95,102,105,110,100,101, 114,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,186,0,0,0,51,4,0,0,115,8,0,0,0,0,1, + 114,186,0,0,0,52,4,0,0,115,8,0,0,0,0,1, 6,1,6,1,14,1,122,23,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,95,105,110,105,116,95,95,99, 1,0,0,0,0,0,0,0,4,0,0,0,3,0,0,0, @@ -1786,7 +1786,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,0,0,0,114,233,0,0,0,218,3,100,111,116,90,2, 109,101,114,2,0,0,0,114,2,0,0,0,114,4,0,0, 0,218,23,95,102,105,110,100,95,112,97,114,101,110,116,95, - 112,97,116,104,95,110,97,109,101,115,57,4,0,0,115,8, + 112,97,116,104,95,110,97,109,101,115,58,4,0,0,115,8, 0,0,0,0,2,18,1,8,2,4,3,122,38,95,78,97, 109,101,115,112,97,99,101,80,97,116,104,46,95,102,105,110, 100,95,112,97,114,101,110,116,95,112,97,116,104,95,110,97, @@ -1799,7 +1799,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,114,101,110,116,95,109,111,100,117,108,101,95,110,97,109, 101,90,14,112,97,116,104,95,97,116,116,114,95,110,97,109, 101,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,244,0,0,0,67,4,0,0,115,4,0,0,0,0,1, + 114,244,0,0,0,68,4,0,0,115,4,0,0,0,0,1, 12,1,122,31,95,78,97,109,101,115,112,97,99,101,80,97, 116,104,46,95,103,101,116,95,112,97,114,101,110,116,95,112, 97,116,104,99,1,0,0,0,0,0,0,0,3,0,0,0, @@ -1815,7 +1815,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,90,11,112,97,114,101,110,116,95,112,97,116,104,114,166, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, 0,0,218,12,95,114,101,99,97,108,99,117,108,97,116,101, - 71,4,0,0,115,16,0,0,0,0,2,12,1,10,1,14, + 72,4,0,0,115,16,0,0,0,0,2,12,1,10,1,14, 3,18,1,6,1,8,1,6,1,122,27,95,78,97,109,101, 115,112,97,99,101,80,97,116,104,46,95,114,101,99,97,108, 99,117,108,97,116,101,99,1,0,0,0,0,0,0,0,1, @@ -1823,7 +1823,7 @@ const unsigned char _Py_M__importlib_external[] = { 116,0,124,0,160,1,161,0,131,1,83,0,41,1,78,41, 2,114,226,0,0,0,114,251,0,0,0,41,1,114,102,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,218,8,95,95,105,116,101,114,95,95,84,4,0,0,115, + 0,218,8,95,95,105,116,101,114,95,95,85,4,0,0,115, 2,0,0,0,0,1,122,23,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,95,105,116,101,114,95,95,99, 3,0,0,0,0,0,0,0,3,0,0,0,3,0,0,0, @@ -1832,14 +1832,14 @@ const unsigned char _Py_M__importlib_external[] = { 0,41,3,114,102,0,0,0,218,5,105,110,100,101,120,114, 35,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, 0,0,0,218,11,95,95,115,101,116,105,116,101,109,95,95, - 87,4,0,0,115,2,0,0,0,0,1,122,26,95,78,97, + 88,4,0,0,115,2,0,0,0,0,1,122,26,95,78,97, 109,101,115,112,97,99,101,80,97,116,104,46,95,95,115,101, 116,105,116,101,109,95,95,99,1,0,0,0,0,0,0,0, 1,0,0,0,3,0,0,0,67,0,0,0,115,12,0,0, 0,116,0,124,0,160,1,161,0,131,1,83,0,41,1,78, 41,2,114,31,0,0,0,114,251,0,0,0,41,1,114,102, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, - 0,0,218,7,95,95,108,101,110,95,95,90,4,0,0,115, + 0,0,218,7,95,95,108,101,110,95,95,91,4,0,0,115, 2,0,0,0,0,1,122,22,95,78,97,109,101,115,112,97, 99,101,80,97,116,104,46,95,95,108,101,110,95,95,99,1, 0,0,0,0,0,0,0,1,0,0,0,3,0,0,0,67, @@ -1848,7 +1848,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,99,101,80,97,116,104,40,123,33,114,125,41,41,2,114, 48,0,0,0,114,243,0,0,0,41,1,114,102,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,218, - 8,95,95,114,101,112,114,95,95,93,4,0,0,115,2,0, + 8,95,95,114,101,112,114,95,95,94,4,0,0,115,2,0, 0,0,0,1,122,23,95,78,97,109,101,115,112,97,99,101, 80,97,116,104,46,95,95,114,101,112,114,95,95,99,2,0, 0,0,0,0,0,0,2,0,0,0,3,0,0,0,67,0, @@ -1856,7 +1856,7 @@ const unsigned char _Py_M__importlib_external[] = { 6,83,0,41,1,78,41,1,114,251,0,0,0,41,2,114, 102,0,0,0,218,4,105,116,101,109,114,2,0,0,0,114, 2,0,0,0,114,4,0,0,0,218,12,95,95,99,111,110, - 116,97,105,110,115,95,95,96,4,0,0,115,2,0,0,0, + 116,97,105,110,115,95,95,97,4,0,0,115,2,0,0,0, 0,1,122,27,95,78,97,109,101,115,112,97,99,101,80,97, 116,104,46,95,95,99,111,110,116,97,105,110,115,95,95,99, 2,0,0,0,0,0,0,0,2,0,0,0,3,0,0,0, @@ -1864,7 +1864,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,161,1,1,0,100,0,83,0,41,1,78,41,2,114,243, 0,0,0,114,165,0,0,0,41,2,114,102,0,0,0,114, 1,1,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,114,165,0,0,0,99,4,0,0,115,2,0,0, + 0,0,0,114,165,0,0,0,100,4,0,0,115,2,0,0, 0,0,1,122,21,95,78,97,109,101,115,112,97,99,101,80, 97,116,104,46,97,112,112,101,110,100,78,41,14,114,107,0, 0,0,114,106,0,0,0,114,108,0,0,0,114,109,0,0, @@ -1872,7 +1872,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,251,0,0,0,114,252,0,0,0,114,254,0,0,0,114, 255,0,0,0,114,0,1,0,0,114,2,1,0,0,114,165, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,2,0, - 0,0,114,4,0,0,0,114,241,0,0,0,44,4,0,0, + 0,0,114,4,0,0,0,114,241,0,0,0,45,4,0,0, 115,20,0,0,0,12,7,8,6,8,10,8,4,8,13,8, 3,8,3,8,3,8,3,8,3,114,241,0,0,0,99,0, 0,0,0,0,0,0,0,0,0,0,0,3,0,0,0,64, @@ -1889,7 +1889,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,243,0,0,0,41,4,114,102,0,0,0,114,100, 0,0,0,114,35,0,0,0,114,247,0,0,0,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,114,186,0,0, - 0,105,4,0,0,115,2,0,0,0,0,1,122,25,95,78, + 0,106,4,0,0,115,2,0,0,0,0,1,122,25,95,78, 97,109,101,115,112,97,99,101,76,111,97,100,101,114,46,95, 95,105,110,105,116,95,95,99,2,0,0,0,0,0,0,0, 2,0,0,0,3,0,0,0,67,0,0,0,115,12,0,0, @@ -1906,21 +1906,21 @@ const unsigned char _Py_M__importlib_external[] = { 2,114,48,0,0,0,114,107,0,0,0,41,2,114,172,0, 0,0,114,191,0,0,0,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,218,11,109,111,100,117,108,101,95,114, - 101,112,114,108,4,0,0,115,2,0,0,0,0,7,122,28, + 101,112,114,109,4,0,0,115,2,0,0,0,0,7,122,28, 95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,114, 46,109,111,100,117,108,101,95,114,101,112,114,99,2,0,0, 0,0,0,0,0,2,0,0,0,1,0,0,0,67,0,0, 0,115,4,0,0,0,100,1,83,0,41,2,78,84,114,2, 0,0,0,41,2,114,102,0,0,0,114,121,0,0,0,114, 2,0,0,0,114,2,0,0,0,114,4,0,0,0,114,161, - 0,0,0,117,4,0,0,115,2,0,0,0,0,1,122,27, + 0,0,0,118,4,0,0,115,2,0,0,0,0,1,122,27, 95,78,97,109,101,115,112,97,99,101,76,111,97,100,101,114, 46,105,115,95,112,97,99,107,97,103,101,99,2,0,0,0, 0,0,0,0,2,0,0,0,1,0,0,0,67,0,0,0, 115,4,0,0,0,100,1,83,0,41,2,78,114,30,0,0, 0,114,2,0,0,0,41,2,114,102,0,0,0,114,121,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,114,202,0,0,0,120,4,0,0,115,2,0,0,0,0, + 0,114,202,0,0,0,121,4,0,0,115,2,0,0,0,0, 1,122,27,95,78,97,109,101,115,112,97,99,101,76,111,97, 100,101,114,46,103,101,116,95,115,111,117,114,99,101,99,2, 0,0,0,0,0,0,0,2,0,0,0,6,0,0,0,67, @@ -1929,7 +1929,7 @@ const unsigned char _Py_M__importlib_external[] = { 122,8,60,115,116,114,105,110,103,62,114,190,0,0,0,84, 41,1,114,204,0,0,0,41,1,114,205,0,0,0,41,2, 114,102,0,0,0,114,121,0,0,0,114,2,0,0,0,114, - 2,0,0,0,114,4,0,0,0,114,188,0,0,0,123,4, + 2,0,0,0,114,4,0,0,0,114,188,0,0,0,124,4, 0,0,115,2,0,0,0,0,1,122,25,95,78,97,109,101, 115,112,97,99,101,76,111,97,100,101,114,46,103,101,116,95, 99,111,100,101,99,2,0,0,0,0,0,0,0,2,0,0, @@ -1939,14 +1939,14 @@ const unsigned char _Py_M__importlib_external[] = { 109,111,100,117,108,101,32,99,114,101,97,116,105,111,110,46, 78,114,2,0,0,0,41,2,114,102,0,0,0,114,166,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,114,187,0,0,0,126,4,0,0,115,0,0,0,0,122, + 0,114,187,0,0,0,127,4,0,0,115,0,0,0,0,122, 30,95,78,97,109,101,115,112,97,99,101,76,111,97,100,101, 114,46,99,114,101,97,116,101,95,109,111,100,117,108,101,99, 2,0,0,0,0,0,0,0,2,0,0,0,1,0,0,0, 67,0,0,0,115,4,0,0,0,100,0,83,0,41,1,78, 114,2,0,0,0,41,2,114,102,0,0,0,114,191,0,0, 0,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,192,0,0,0,129,4,0,0,115,2,0,0,0,0,1, + 114,192,0,0,0,130,4,0,0,115,2,0,0,0,0,1, 122,28,95,78,97,109,101,115,112,97,99,101,76,111,97,100, 101,114,46,101,120,101,99,95,109,111,100,117,108,101,99,2, 0,0,0,0,0,0,0,2,0,0,0,4,0,0,0,67, @@ -1964,7 +1964,7 @@ const unsigned char _Py_M__importlib_external[] = { 41,4,114,116,0,0,0,114,130,0,0,0,114,243,0,0, 0,114,193,0,0,0,41,2,114,102,0,0,0,114,121,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,114,194,0,0,0,132,4,0,0,115,6,0,0,0,0, + 0,114,194,0,0,0,133,4,0,0,115,6,0,0,0,0, 7,6,1,8,1,122,28,95,78,97,109,101,115,112,97,99, 101,76,111,97,100,101,114,46,108,111,97,100,95,109,111,100, 117,108,101,78,41,12,114,107,0,0,0,114,106,0,0,0, @@ -1972,7 +1972,7 @@ const unsigned char _Py_M__importlib_external[] = { 4,1,0,0,114,161,0,0,0,114,202,0,0,0,114,188, 0,0,0,114,187,0,0,0,114,192,0,0,0,114,194,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,2,0,0, - 0,114,4,0,0,0,114,3,1,0,0,104,4,0,0,115, + 0,114,4,0,0,0,114,3,1,0,0,105,4,0,0,115, 16,0,0,0,8,1,8,3,12,9,8,3,8,3,8,3, 8,3,8,3,114,3,1,0,0,99,0,0,0,0,0,0, 0,0,0,0,0,0,4,0,0,0,64,0,0,0,115,106, @@ -2006,7 +2006,7 @@ const unsigned char _Py_M__importlib_external[] = { 218,6,118,97,108,117,101,115,114,110,0,0,0,114,6,1, 0,0,41,2,114,172,0,0,0,218,6,102,105,110,100,101, 114,114,2,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,6,1,0,0,150,4,0,0,115,6,0,0,0,0,4, + 114,6,1,0,0,151,4,0,0,115,6,0,0,0,0,4, 16,1,10,1,122,28,80,97,116,104,70,105,110,100,101,114, 46,105,110,118,97,108,105,100,97,116,101,95,99,97,99,104, 101,115,99,2,0,0,0,0,0,0,0,3,0,0,0,9, @@ -2026,7 +2026,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,3,114,172,0,0,0,114,35,0,0,0,90, 4,104,111,111,107,114,2,0,0,0,114,2,0,0,0,114, 4,0,0,0,218,11,95,112,97,116,104,95,104,111,111,107, - 115,158,4,0,0,115,16,0,0,0,0,3,16,1,12,1, + 115,159,4,0,0,115,16,0,0,0,0,3,16,1,12,1, 12,1,2,1,8,1,14,1,12,2,122,22,80,97,116,104, 70,105,110,100,101,114,46,95,112,97,116,104,95,104,111,111, 107,115,99,2,0,0,0,0,0,0,0,3,0,0,0,8, @@ -2056,7 +2056,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,111,114,114,11,1,0,0,41,3,114,172,0,0,0,114, 35,0,0,0,114,9,1,0,0,114,2,0,0,0,114,2, 0,0,0,114,4,0,0,0,218,20,95,112,97,116,104,95, - 105,109,112,111,114,116,101,114,95,99,97,99,104,101,171,4, + 105,109,112,111,114,116,101,114,95,99,97,99,104,101,172,4, 0,0,115,22,0,0,0,0,8,8,1,2,1,12,1,14, 3,6,1,2,1,14,1,14,1,10,1,16,1,122,31,80, 97,116,104,70,105,110,100,101,114,46,95,112,97,116,104,95, @@ -2074,7 +2074,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,9,1,0,0,114,122,0,0,0,114,123,0, 0,0,114,166,0,0,0,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,218,16,95,108,101,103,97,99,121,95, - 103,101,116,95,115,112,101,99,193,4,0,0,115,18,0,0, + 103,101,116,95,115,112,101,99,194,4,0,0,115,18,0,0, 0,0,4,10,1,16,2,10,1,4,1,8,1,12,1,12, 1,6,1,122,27,80,97,116,104,70,105,110,100,101,114,46, 95,108,101,103,97,99,121,95,103,101,116,95,115,112,101,99, @@ -2105,7 +2105,7 @@ const unsigned char _Py_M__importlib_external[] = { 110,97,109,101,115,112,97,99,101,95,112,97,116,104,90,5, 101,110,116,114,121,114,9,1,0,0,114,166,0,0,0,114, 123,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,218,9,95,103,101,116,95,115,112,101,99,208,4, + 0,0,0,218,9,95,103,101,116,95,115,112,101,99,209,4, 0,0,115,40,0,0,0,0,5,4,1,10,1,14,1,2, 1,10,1,8,1,10,1,14,2,12,1,8,1,2,1,10, 1,4,1,6,1,8,1,8,5,14,2,12,1,6,1,122, @@ -2133,7 +2133,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,241,0,0,0,41,6,114,172,0,0,0,114,121,0,0, 0,114,35,0,0,0,114,181,0,0,0,114,166,0,0,0, 114,16,1,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,182,0,0,0,240,4,0,0,115,26,0, + 4,0,0,0,114,182,0,0,0,241,4,0,0,115,26,0, 0,0,0,6,8,1,6,1,14,1,8,1,4,1,10,1, 6,1,4,3,6,1,16,1,4,2,6,2,122,20,80,97, 116,104,70,105,110,100,101,114,46,102,105,110,100,95,115,112, @@ -2154,7 +2154,7 @@ const unsigned char _Py_M__importlib_external[] = { 32,32,32,32,32,32,32,32,78,41,2,114,182,0,0,0, 114,122,0,0,0,41,4,114,172,0,0,0,114,121,0,0, 0,114,35,0,0,0,114,166,0,0,0,114,2,0,0,0, - 114,2,0,0,0,114,4,0,0,0,114,183,0,0,0,8, + 114,2,0,0,0,114,4,0,0,0,114,183,0,0,0,9, 5,0,0,115,8,0,0,0,0,8,12,1,8,1,4,1, 122,22,80,97,116,104,70,105,110,100,101,114,46,102,105,110, 100,95,109,111,100,117,108,101,41,1,78,41,2,78,78,41, @@ -2163,7 +2163,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,114,11,1,0,0,114,13,1,0,0,114,14,1,0, 0,114,17,1,0,0,114,182,0,0,0,114,183,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,5,1,0,0,146,4,0,0,115,20,0, + 4,0,0,0,114,5,1,0,0,147,4,0,0,115,20,0, 0,0,12,4,12,8,12,13,12,22,12,15,2,1,12,31, 2,1,12,23,2,1,114,5,1,0,0,99,0,0,0,0, 0,0,0,0,0,0,0,0,3,0,0,0,64,0,0,0, @@ -2207,7 +2207,7 @@ const unsigned char _Py_M__importlib_external[] = { 2,86,0,1,0,113,2,100,0,83,0,41,1,78,114,2, 0,0,0,41,2,114,22,0,0,0,114,236,0,0,0,41, 1,114,122,0,0,0,114,2,0,0,0,114,4,0,0,0, - 114,238,0,0,0,37,5,0,0,115,2,0,0,0,4,0, + 114,238,0,0,0,38,5,0,0,115,2,0,0,0,4,0, 122,38,70,105,108,101,70,105,110,100,101,114,46,95,95,105, 110,105,116,95,95,46,60,108,111,99,97,108,115,62,46,60, 103,101,110,101,120,112,114,62,114,59,0,0,0,114,89,0, @@ -2219,7 +2219,7 @@ const unsigned char _Py_M__importlib_external[] = { 102,0,0,0,114,35,0,0,0,218,14,108,111,97,100,101, 114,95,100,101,116,97,105,108,115,90,7,108,111,97,100,101, 114,115,114,168,0,0,0,114,2,0,0,0,41,1,114,122, - 0,0,0,114,4,0,0,0,114,186,0,0,0,31,5,0, + 0,0,0,114,4,0,0,0,114,186,0,0,0,32,5,0, 0,115,16,0,0,0,0,4,4,1,14,1,28,1,6,2, 10,1,6,1,8,1,122,19,70,105,108,101,70,105,110,100, 101,114,46,95,95,105,110,105,116,95,95,99,1,0,0,0, @@ -2229,7 +2229,7 @@ const unsigned char _Py_M__importlib_external[] = { 101,32,100,105,114,101,99,116,111,114,121,32,109,116,105,109, 101,46,114,89,0,0,0,78,41,1,114,20,1,0,0,41, 1,114,102,0,0,0,114,2,0,0,0,114,2,0,0,0, - 114,4,0,0,0,114,6,1,0,0,45,5,0,0,115,2, + 114,4,0,0,0,114,6,1,0,0,46,5,0,0,115,2, 0,0,0,0,2,122,28,70,105,108,101,70,105,110,100,101, 114,46,105,110,118,97,108,105,100,97,116,101,95,99,97,99, 104,101,115,99,2,0,0,0,0,0,0,0,3,0,0,0, @@ -2252,7 +2252,7 @@ const unsigned char _Py_M__importlib_external[] = { 78,41,3,114,182,0,0,0,114,122,0,0,0,114,158,0, 0,0,41,3,114,102,0,0,0,114,121,0,0,0,114,166, 0,0,0,114,2,0,0,0,114,2,0,0,0,114,4,0, - 0,0,114,119,0,0,0,51,5,0,0,115,8,0,0,0, + 0,0,114,119,0,0,0,52,5,0,0,115,8,0,0,0, 0,7,10,1,8,1,8,1,122,22,70,105,108,101,70,105, 110,100,101,114,46,102,105,110,100,95,108,111,97,100,101,114, 99,6,0,0,0,0,0,0,0,7,0,0,0,6,0,0, @@ -2263,7 +2263,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,114,167,0,0,0,114,121,0,0,0,114,35,0,0,0, 90,4,115,109,115,108,114,181,0,0,0,114,122,0,0,0, 114,2,0,0,0,114,2,0,0,0,114,4,0,0,0,114, - 17,1,0,0,63,5,0,0,115,6,0,0,0,0,1,10, + 17,1,0,0,64,5,0,0,115,6,0,0,0,0,1,10, 1,8,1,122,20,70,105,108,101,70,105,110,100,101,114,46, 95,103,101,116,95,115,112,101,99,78,99,3,0,0,0,0, 0,0,0,14,0,0,0,8,0,0,0,67,0,0,0,115, @@ -2317,7 +2317,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,167,0,0,0,90,13,105,110,105,116,95,102,105,108,101, 110,97,109,101,90,9,102,117,108,108,95,112,97,116,104,114, 166,0,0,0,114,2,0,0,0,114,2,0,0,0,114,4, - 0,0,0,114,182,0,0,0,68,5,0,0,115,70,0,0, + 0,0,0,114,182,0,0,0,69,5,0,0,115,70,0,0, 0,0,5,4,1,14,1,2,1,24,1,14,1,10,1,10, 1,8,1,6,2,6,1,6,1,10,2,6,1,4,2,8, 1,12,1,16,1,8,1,10,1,8,1,24,4,8,2,16, @@ -2349,7 +2349,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,2,0,0,0,41,1,114,90,0,0,0,41,2,114,22, 0,0,0,90,2,102,110,114,2,0,0,0,114,2,0,0, 0,114,4,0,0,0,250,9,60,115,101,116,99,111,109,112, - 62,145,5,0,0,115,2,0,0,0,6,0,122,41,70,105, + 62,146,5,0,0,115,2,0,0,0,6,0,122,41,70,105, 108,101,70,105,110,100,101,114,46,95,102,105,108,108,95,99, 97,99,104,101,46,60,108,111,99,97,108,115,62,46,60,115, 101,116,99,111,109,112,62,78,41,18,114,35,0,0,0,114, @@ -2365,7 +2365,7 @@ const unsigned char _Py_M__importlib_external[] = { 111,110,116,101,110,116,115,114,1,1,0,0,114,100,0,0, 0,114,248,0,0,0,114,236,0,0,0,90,8,110,101,119, 95,110,97,109,101,114,2,0,0,0,114,2,0,0,0,114, - 4,0,0,0,114,25,1,0,0,116,5,0,0,115,34,0, + 4,0,0,0,114,25,1,0,0,117,5,0,0,115,34,0, 0,0,0,2,6,1,2,1,22,1,20,3,10,3,12,1, 12,7,6,1,10,1,16,1,4,1,18,2,4,1,14,1, 6,1,12,1,122,22,70,105,108,101,70,105,110,100,101,114, @@ -2403,7 +2403,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,41,1,114,35,0,0,0,41,2,114,172,0,0, 0,114,24,1,0,0,114,2,0,0,0,114,4,0,0,0, 218,24,112,97,116,104,95,104,111,111,107,95,102,111,114,95, - 70,105,108,101,70,105,110,100,101,114,157,5,0,0,115,6, + 70,105,108,101,70,105,110,100,101,114,158,5,0,0,115,6, 0,0,0,0,2,8,1,12,1,122,54,70,105,108,101,70, 105,110,100,101,114,46,112,97,116,104,95,104,111,111,107,46, 60,108,111,99,97,108,115,62,46,112,97,116,104,95,104,111, @@ -2411,7 +2411,7 @@ const unsigned char _Py_M__importlib_external[] = { 114,114,2,0,0,0,41,3,114,172,0,0,0,114,24,1, 0,0,114,30,1,0,0,114,2,0,0,0,41,2,114,172, 0,0,0,114,24,1,0,0,114,4,0,0,0,218,9,112, - 97,116,104,95,104,111,111,107,147,5,0,0,115,4,0,0, + 97,116,104,95,104,111,111,107,148,5,0,0,115,4,0,0, 0,0,10,14,6,122,20,70,105,108,101,70,105,110,100,101, 114,46,112,97,116,104,95,104,111,111,107,99,1,0,0,0, 0,0,0,0,1,0,0,0,3,0,0,0,67,0,0,0, @@ -2419,7 +2419,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,41,2,78,122,16,70,105,108,101,70,105,110,100,101,114, 40,123,33,114,125,41,41,2,114,48,0,0,0,114,35,0, 0,0,41,1,114,102,0,0,0,114,2,0,0,0,114,2, - 0,0,0,114,4,0,0,0,114,0,1,0,0,165,5,0, + 0,0,0,114,4,0,0,0,114,0,1,0,0,166,5,0, 0,115,2,0,0,0,0,1,122,19,70,105,108,101,70,105, 110,100,101,114,46,95,95,114,101,112,114,95,95,41,1,78, 41,15,114,107,0,0,0,114,106,0,0,0,114,108,0,0, @@ -2428,7 +2428,7 @@ const unsigned char _Py_M__importlib_external[] = { 17,1,0,0,114,182,0,0,0,114,25,1,0,0,114,184, 0,0,0,114,31,1,0,0,114,0,1,0,0,114,2,0, 0,0,114,2,0,0,0,114,2,0,0,0,114,4,0,0, - 0,114,18,1,0,0,22,5,0,0,115,18,0,0,0,12, + 0,114,18,1,0,0,23,5,0,0,115,18,0,0,0,12, 9,8,14,8,4,4,2,8,12,8,5,10,48,8,31,12, 18,114,18,1,0,0,99,4,0,0,0,0,0,0,0,6, 0,0,0,8,0,0,0,67,0,0,0,115,146,0,0,0, @@ -2451,7 +2451,7 @@ const unsigned char _Py_M__importlib_external[] = { 97,109,101,90,9,99,112,97,116,104,110,97,109,101,114,122, 0,0,0,114,166,0,0,0,114,2,0,0,0,114,2,0, 0,0,114,4,0,0,0,218,14,95,102,105,120,95,117,112, - 95,109,111,100,117,108,101,171,5,0,0,115,34,0,0,0, + 95,109,111,100,117,108,101,172,5,0,0,115,34,0,0,0, 0,2,10,1,10,1,4,1,4,1,8,1,8,1,12,2, 10,1,4,1,14,1,2,1,8,1,8,1,8,1,12,1, 14,2,114,36,1,0,0,99,0,0,0,0,0,0,0,0, @@ -2471,7 +2471,7 @@ const unsigned char _Py_M__importlib_external[] = { 90,10,101,120,116,101,110,115,105,111,110,115,90,6,115,111, 117,114,99,101,90,8,98,121,116,101,99,111,100,101,114,2, 0,0,0,114,2,0,0,0,114,4,0,0,0,114,163,0, - 0,0,194,5,0,0,115,8,0,0,0,0,5,12,1,8, + 0,0,195,5,0,0,115,8,0,0,0,0,5,12,1,8, 1,8,1,114,163,0,0,0,99,1,0,0,0,0,0,0, 0,12,0,0,0,9,0,0,0,67,0,0,0,115,156,1, 0,0,124,0,97,0,116,0,106,1,97,1,116,0,106,2, @@ -2521,7 +2521,7 @@ const unsigned char _Py_M__importlib_external[] = { 1,100,0,107,2,86,0,1,0,113,2,100,1,83,0,41, 2,114,29,0,0,0,78,41,1,114,31,0,0,0,41,2, 114,22,0,0,0,114,79,0,0,0,114,2,0,0,0,114, - 2,0,0,0,114,4,0,0,0,114,238,0,0,0,230,5, + 2,0,0,0,114,4,0,0,0,114,238,0,0,0,231,5, 0,0,115,2,0,0,0,4,0,122,25,95,115,101,116,117, 112,46,60,108,111,99,97,108,115,62,46,60,103,101,110,101, 120,112,114,62,114,60,0,0,0,122,30,105,109,112,111,114, @@ -2549,7 +2549,7 @@ const unsigned char _Py_M__importlib_external[] = { 119,101,97,107,114,101,102,95,109,111,100,117,108,101,90,13, 119,105,110,114,101,103,95,109,111,100,117,108,101,114,2,0, 0,0,114,2,0,0,0,114,4,0,0,0,218,6,95,115, - 101,116,117,112,205,5,0,0,115,76,0,0,0,0,8,4, + 101,116,117,112,206,5,0,0,115,76,0,0,0,0,8,4, 1,6,1,6,3,10,1,10,1,10,1,12,2,10,1,16, 3,22,1,14,2,22,1,8,1,10,1,10,1,4,2,2, 1,10,1,6,1,14,1,12,2,8,1,12,1,12,1,18, @@ -2569,7 +2569,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,41,2,114,42,1,0,0,90,17,115,117,112,112,111, 114,116,101,100,95,108,111,97,100,101,114,115,114,2,0,0, 0,114,2,0,0,0,114,4,0,0,0,218,8,95,105,110, - 115,116,97,108,108,13,6,0,0,115,8,0,0,0,0,2, + 115,116,97,108,108,14,6,0,0,115,8,0,0,0,0,2, 8,1,6,1,20,1,114,45,1,0,0,41,1,114,47,0, 0,0,41,1,78,41,3,78,78,78,41,2,114,60,0,0, 0,114,60,0,0,0,41,1,84,41,1,78,41,1,78,41, @@ -2601,7 +2601,7 @@ const unsigned char _Py_M__importlib_external[] = { 0,0,0,114,2,0,0,0,114,4,0,0,0,218,8,60, 109,111,100,117,108,101,62,23,0,0,0,115,116,0,0,0, 4,0,4,1,4,1,2,1,6,3,8,17,8,5,8,5, - 8,6,8,12,8,10,8,9,8,5,8,7,10,22,10,126, + 8,6,8,12,8,10,8,9,8,5,8,7,10,22,10,127, 16,1,12,2,4,1,4,2,6,2,6,2,8,2,16,45, 8,34,8,19,8,12,8,12,8,28,8,17,8,33,8,28, 8,24,10,13,10,10,10,11,8,14,6,3,4,1,14,67, diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 567f8723a678..da6d032bce41 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -126,7 +126,7 @@ static void *opcode_targets[256] = { &&TARGET_LOAD_FAST, &&TARGET_STORE_FAST, &&TARGET_DELETE_FAST, - &&TARGET_STORE_ANNOTATION, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&TARGET_RAISE_VARARGS,

1 0

bpo-27931: Fix email address header parsing error (GH-5329) (GH-5431)
by Mariatta Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/6ea75b174da0cf824e2acc5db6b53798f5… commit: 6ea75b174da0cf824e2acc5db6b53798f5f4e4f9 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Mariatta <Mariatta(a)users.noreply.github.com> date: 2018-01-29T14:27:55-08:00 summary: bpo-27931: Fix email address header parsing error (GH-5329) (GH-5431) Correctly handle addresses whose username is an empty quoted string. (cherry picked from commit aa218d1649690d1c1ba86a9972f7fae646bf1a8f) Co-authored-by: jayyyin <jayyin11043(a)hotmail.com> files: A Misc/NEWS.d/next/Library/2018-01-25-21-04-11.bpo-27931.e4r52t.rst M Lib/email/_header_value_parser.py M Lib/test/test_email/test__header_value_parser.py diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 3ebbbe5383ab..b23c897e97bb 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -430,7 +430,10 @@ def route(self): def addr_spec(self): for x in self: if x.token_type == 'addr-spec': - return x.addr_spec + if x.local_part: + return x.addr_spec + else: + return quote_string(x.local_part) + x.addr_spec else: return '<>' @@ -1165,6 +1168,9 @@ def get_bare_quoted_string(value): "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() value = value[1:] + if value[0] == '"': + token, value = get_qcontent(value) + bare_quoted_string.append(token) while value and value[0] != '"': if value[0] in WSP: token, value = get_fws(value) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index 1667617b9e46..5cdc4bcecad4 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -490,6 +490,10 @@ def test_get_bare_quoted_string_must_start_with_dquote(self): with self.assertRaises(errors.HeaderParseError): parser.get_bare_quoted_string(' "foo"') + def test_get_bare_quoted_string_only_quotes(self): + self._test_get_x(parser.get_bare_quoted_string, + '""', '""', '', [], '') + def test_get_bare_quoted_string_following_wsp_preserved(self): self._test_get_x(parser.get_bare_quoted_string, '"foo"\t bar', '"foo"', 'foo', [], '\t bar') @@ -1467,6 +1471,19 @@ def test_get_angle_addr_empty(self): self.assertIsNone(angle_addr.route) self.assertEqual(angle_addr.addr_spec, '<>') + def test_get_angle_addr_qs_only_quotes(self): + angle_addr = self._test_get_x(parser.get_angle_addr, + '<""@example.com>', + '<""@example.com>', + '<""@example.com>', + [], + '') + self.assertEqual(angle_addr.token_type, 'angle-addr') + self.assertEqual(angle_addr.local_part, '') + self.assertEqual(angle_addr.domain, 'example.com') + self.assertIsNone(angle_addr.route) + self.assertEqual(angle_addr.addr_spec, '""@example.com') + def test_get_angle_addr_with_cfws(self): angle_addr = self._test_get_x(parser.get_angle_addr, ' (foo) <dinsdale(a)example.com>(bar)', diff --git a/Misc/NEWS.d/next/Library/2018-01-25-21-04-11.bpo-27931.e4r52t.rst b/Misc/NEWS.d/next/Library/2018-01-25-21-04-11.bpo-27931.e4r52t.rst new file mode 100644 index 000000000000..7324247357a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-01-25-21-04-11.bpo-27931.e4r52t.rst @@ -0,0 +1 @@ +Fix email address header parsing error when the username is an empty quoted string. Patch by Xiang Zhang.

1 0

bpo-28134: Auto-detect socket values from file descriptor (#1349)
by Christian Heimes Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/b6e43af669f61a37a29d8ff0785455108e… commit: b6e43af669f61a37a29d8ff0785455108e6bc29d branch: master author: Christian Heimes <christian(a)python.org> committer: GitHub <noreply(a)github.com> date: 2018-01-29T22:37:58+01:00 summary: bpo-28134: Auto-detect socket values from file descriptor (#1349) Fix socket(fileno=fd) by auto-detecting the socket's family, type, and proto from the file descriptor. The auto-detection can be overruled by passing in family, type, and proto explicitly. Without the fix, all socket except for TCP/IP over IPv4 are basically broken: >>> s = socket.create_connection(('www.python.org', 443)) >>> s <socket.socket fd=3, family=AddressFamily.AF_INET6, type=SocketKind.SOCK_STREAM, proto=6, laddr=('2003:58:bc4a:3b00:56ee:75ff:fe47:ca7b', 59730, 0, 0), raddr=('2a04:4e42:1b::223', 443, 0, 0)> >>> socket.socket(fileno=s.fileno()) <socket.socket fd=3, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=0, laddr=('2003:58:bc4a:3b00::%2550471192', 59730, 0, 2550471192), raddr=('2a04:4e42:1b:0:700c:e70b:ff7f:0%2550471192', 443, 0, 2550471192)> Signed-off-by: Christian Heimes <christian(a)python.org> files: A Misc/NEWS.d/next/Library/2017-12-24-20-01-09.bpo-28134.HJ8Beb.rst M Doc/library/socket.rst M Lib/socket.py M Lib/test/_test_multiprocessing.py M Lib/test/test_socket.py M Modules/socketmodule.c diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index faa260e5d6c5..7f0d4ede7ccf 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -461,10 +461,15 @@ The following functions all create :ref:`socket objects <socket-objects>`. :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other ``SOCK_`` constants. The protocol number is usually zero and may be omitted or in the case where the address family is :const:`AF_CAN` the protocol should be one - of :const:`CAN_RAW`, :const:`CAN_BCM` or :const:`CAN_ISOTP`. If *fileno* is specified, the other - arguments are ignored, causing the socket with the specified file descriptor - to return. Unlike :func:`socket.fromfd`, *fileno* will return the same - socket and not a duplicate. This may help close a detached socket using + of :const:`CAN_RAW`, :const:`CAN_BCM` or :const:`CAN_ISOTP` + + If *fileno* is specified, the values for *family*, *type*, and *proto* are + auto-detected from the specified file descriptor. Auto-detection can be + overruled by calling the function with explicit *family*, *type*, or *proto* + arguments. This only affects how Python represents e.g. the return value + of :meth:`socket.getpeername` but not the actual OS resource. Unlike + :func:`socket.fromfd`, *fileno* will return the same socket and not a + duplicate. This may help close a detached socket using :meth:`socket.close()`. The newly created socket is :ref:`non-inheritable <fd_inheritance>`. diff --git a/Lib/socket.py b/Lib/socket.py index 2d8aee3e9047..cfa605a22ada 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -136,11 +136,18 @@ class socket(_socket.socket): __slots__ = ["__weakref__", "_io_refs", "_closed"] - def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None): + def __init__(self, family=-1, type=-1, proto=-1, fileno=None): # For user code address family and type values are IntEnum members, but # for the underlying _socket.socket they're just integers. The # constructor of _socket.socket converts the given argument to an # integer automatically. + if fileno is None: + if family == -1: + family = AF_INET + if type == -1: + type = SOCK_STREAM + if proto == -1: + proto = 0 _socket.socket.__init__(self, family, type, proto, fileno) self._io_refs = 0 self._closed = False diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 05166b91ba83..1e497a572a76 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -4204,7 +4204,7 @@ def get_high_socket_fd(self): def close(self, fd): if WIN32: - socket.socket(fileno=fd).close() + socket.socket(socket.AF_INET, socket.SOCK_STREAM, fileno=fd).close() else: os.close(fd) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 365da36c4210..89cf1fa52009 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -20,6 +20,7 @@ import pickle import struct import random +import shutil import string import _thread as thread import threading @@ -1284,6 +1285,7 @@ def testSendAfterClose(self): def testCloseException(self): sock = socket.socket() + sock.bind((socket._LOCALHOST, 0)) socket.socket(fileno=sock.fileno()).close() try: sock.close() @@ -1636,9 +1638,11 @@ def test_uknown_socket_family_repr(self): ) + 1 with socket.socket( - family=unknown_family, type=unknown_type, fileno=fd) as s: + family=unknown_family, type=unknown_type, proto=23, + fileno=fd) as s: self.assertEqual(s.family, unknown_family) self.assertEqual(s.type, unknown_type) + self.assertEqual(s.proto, 23) @unittest.skipUnless(hasattr(os, 'sendfile'), 'test needs os.sendfile()') def test__sendfile_use_sendfile(self): @@ -1658,6 +1662,45 @@ def fileno(self): with self.assertRaises(TypeError): sock._sendfile_use_sendfile(File(None)) + def _test_socket_fileno(self, s, family, stype): + self.assertEqual(s.family, family) + self.assertEqual(s.type, stype) + + fd = s.fileno() + s2 = socket.socket(fileno=fd) + self.addCleanup(s2.close) + # detach old fd to avoid double close + s.detach() + self.assertEqual(s2.family, family) + self.assertEqual(s2.type, stype) + self.assertEqual(s2.fileno(), fd) + + def test_socket_fileno(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(s.close) + s.bind((support.HOST, 0)) + self._test_socket_fileno(s, socket.AF_INET, socket.SOCK_STREAM) + + if hasattr(socket, "SOCK_DGRAM"): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + s.bind((support.HOST, 0)) + self._test_socket_fileno(s, socket.AF_INET, socket.SOCK_DGRAM) + + if support.IPV6_ENABLED: + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self.addCleanup(s.close) + s.bind((support.HOSTv6, 0, 0, 0)) + self._test_socket_fileno(s, socket.AF_INET6, socket.SOCK_STREAM) + + if hasattr(socket, "AF_UNIX"): + tmpdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmpdir) + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self.addCleanup(s.close) + s.bind(os.path.join(tmpdir, 'socket')) + self._test_socket_fileno(s, socket.AF_UNIX, socket.SOCK_STREAM) + @unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') class BasicCANTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2017-12-24-20-01-09.bpo-28134.HJ8Beb.rst b/Misc/NEWS.d/next/Library/2017-12-24-20-01-09.bpo-28134.HJ8Beb.rst new file mode 100644 index 000000000000..9c4c683db6e3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-24-20-01-09.bpo-28134.HJ8Beb.rst @@ -0,0 +1,2 @@ +Sockets now auto-detect family, type and protocol from file descriptor by +default. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 8744ea0be87c..8cc5d14cbb20 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -102,7 +102,8 @@ Local naming conventions: /* Socket object documentation */ PyDoc_STRVAR(sock_doc, -"socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) -> socket object\n\ +"socket(family=AF_INET, type=SOCK_STREAM, proto=0) -> socket object\n\ +socket(family=-1, type=-1, proto=-1, fileno=None) -> socket object\n\ \n\ Open a socket of the given type. The family argument specifies the\n\ address family; it defaults to AF_INET. The type argument specifies\n\ @@ -111,6 +112,9 @@ or datagram (SOCK_DGRAM) socket. The protocol argument defaults to 0,\n\ specifying the default protocol. Keyword arguments are accepted.\n\ The socket is created as non-inheritable.\n\ \n\ +When a fileno is passed in, family, type and proto are auto-detected,\n\ +unless they are explicitly set.\n\ +\n\ A socket object represents one endpoint of a network connection.\n\ \n\ Methods of socket objects (keyword arguments not allowed):\n\ @@ -4792,7 +4796,7 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwds) PySocketSockObject *s = (PySocketSockObject *)self; PyObject *fdobj = NULL; SOCKET_T fd = INVALID_SOCKET; - int family = AF_INET, type = SOCK_STREAM, proto = 0; + int family = -1, type = -1, proto = -1; static char *keywords[] = {"family", "type", "proto", "fileno", 0}; #ifndef MS_WINDOWS #ifdef SOCK_CLOEXEC @@ -4842,9 +4846,72 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwds) "can't use invalid socket value"); return -1; } + + if (family == -1) { + sock_addr_t addrbuf; + socklen_t addrlen = sizeof(sock_addr_t); + + memset(&addrbuf, 0, addrlen); + if (getsockname(fd, SAS2SA(&addrbuf), &addrlen) == 0) { + family = SAS2SA(&addrbuf)->sa_family; + } else { +#ifdef MS_WINDOWS + PyErr_SetFromWindowsErrWithFilename(0, "family"); +#else + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "family"); +#endif + return -1; + } + } +#ifdef SO_TYPE + if (type == -1) { + int tmp; + socklen_t slen = sizeof(tmp); + if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &tmp, &slen) == 0) { + type = tmp; + } else { +#ifdef MS_WINDOWS + PyErr_SetFromWindowsErrWithFilename(0, "type"); +#else + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "type"); +#endif + return -1; + } + } +#else + type = SOCK_STREAM; +#endif +#ifdef SO_PROTOCOL + if (proto == -1) { + int tmp; + socklen_t slen = sizeof(tmp); + if (getsockopt(fd, SOL_SOCKET, SO_PROTOCOL, &tmp, &slen) == 0) { + proto = tmp; + } else { +#ifdef MS_WINDOWS + PyErr_SetFromWindowsErrWithFilename(0, "protocol"); +#else + PyErr_SetFromErrnoWithFilename(PyExc_OSError, "protocol"); +#endif + return -1; + } + } +#else + proto = 0; +#endif } } else { + /* No fd, default to AF_INET and SOCK_STREAM */ + if (family == -1) { + family = AF_INET; + } + if (type == -1) { + type = SOCK_STREAM; + } + if (proto == -1) { + proto = 0; + } #ifdef MS_WINDOWS /* Windows implementation */ #ifndef WSA_FLAG_NO_HANDLE_INHERIT

1 0

bpo-31356: Add context manager to temporarily disable GC (GH-4224)
by Raymond Hettinger Jan. 29, 2018

Jan. 29, 2018

https://github.com/python/cpython/commit/72a0d218dcc94a3cc409a9ef32dfcd5a7b… commit: 72a0d218dcc94a3cc409a9ef32dfcd5a7bbcb43c branch: master author: Pablo Galindo <Pablogsal(a)gmail.com> committer: Raymond Hettinger <rhettinger(a)users.noreply.github.com> date: 2018-01-29T12:37:09-08:00 summary: bpo-31356: Add context manager to temporarily disable GC (GH-4224) files: A Misc/NEWS.d/next/Core and Builtins/2017-11-02-00-34-42.bpo-31356.54Lb8U.rst M Doc/library/gc.rst M Include/internal/mem.h M Lib/test/test_gc.py M Modules/gcmodule.c diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 153d8fb70456..92240c760677 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -33,6 +33,34 @@ The :mod:`gc` module provides the following functions: Disable automatic garbage collection. +.. class:: ensure_disabled() + + Return a context manager object that disables the garbage collector and reenables the previous + state upon completion of the block. This is basically equivalent to:: + + from gc import enable, disable, isenabled + + @contextmanager + def ensure_disabled(): + was_enabled_previously = isenabled() + gc.disable() + yield + if was_enabled_previously: + gc.enable() + + And lets you write code like this:: + + with ensure_disabled(): + run_some_timing() + + with ensure_disabled(): + # do_something_that_has_real_time_guarantees + # such as a pair trade, robotic braking, etc + + without needing to explicitly enable and disable the garbage collector yourself. + This context manager is implemented in C to assure atomicity, thread safety and speed. + + .. function:: isenabled() Returns true if automatic collection is enabled. diff --git a/Include/internal/mem.h b/Include/internal/mem.h index a731e30e6af7..4a84b4a78d90 100644 --- a/Include/internal/mem.h +++ b/Include/internal/mem.h @@ -116,6 +116,7 @@ struct _gc_runtime_state { int enabled; int debug; + long disabled_threads; /* linked lists of container objects */ struct gc_generation generations[NUM_GENERATIONS]; PyGC_Head *generation0; diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 904fc7d88c03..246980aa74c2 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,7 +1,7 @@ import unittest from test.support import (verbose, refcount_test, run_unittest, strip_python_stderr, cpython_only, start_threads, - temp_dir, requires_type_collecting) + temp_dir, requires_type_collecting,reap_threads) from test.support.script_helper import assert_python_ok, make_script import sys @@ -9,6 +9,8 @@ import gc import weakref import threading +import warnings + try: from _testcapi import with_tp_del @@ -1007,6 +1009,73 @@ def __del__(self): # empty __dict__. self.assertEqual(x, None) + def test_ensure_disabled(self): + original_status = gc.isenabled() + + with gc.ensure_disabled(): + inside_status = gc.isenabled() + + after_status = gc.isenabled() + self.assertEqual(original_status, True) + self.assertEqual(inside_status, False) + self.assertEqual(after_status, True) + + def test_ensure_disabled_with_gc_disabled(self): + gc.disable() + + original_status = gc.isenabled() + + with gc.ensure_disabled(): + inside_status = gc.isenabled() + + after_status = gc.isenabled() + self.assertEqual(original_status, False) + self.assertEqual(inside_status, False) + self.assertEqual(after_status, False) + + @reap_threads + def test_ensure_disabled_thread(self): + + thread_original_status = None + thread_inside_status = None + thread_after_status = None + + def disabling_thread(): + nonlocal thread_original_status + nonlocal thread_inside_status + nonlocal thread_after_status + thread_original_status = gc.isenabled() + + with gc.ensure_disabled(): + time.sleep(0.01) + thread_inside_status = gc.isenabled() + + thread_after_status = gc.isenabled() + + original_status = gc.isenabled() + + with warnings.catch_warnings(record=True) as w, gc.ensure_disabled(): + inside_status_before_thread = gc.isenabled() + thread = threading.Thread(target=disabling_thread) + thread.start() + inside_status_after_thread = gc.isenabled() + + after_status = gc.isenabled() + thread.join() + + self.assertEqual(len(w), 1) + self.assertTrue(issubclass(w[-1].category, RuntimeWarning)) + self.assertEqual("Garbage collector enabled while another thread is " + "inside gc.ensure_enabled", str(w[-1].message)) + self.assertEqual(original_status, True) + self.assertEqual(inside_status_before_thread, False) + self.assertEqual(thread_original_status, False) + self.assertEqual(thread_inside_status, True) + self.assertEqual(thread_after_status, False) + self.assertEqual(inside_status_after_thread, False) + self.assertEqual(after_status, True) + + def test_main(): enabled = gc.isenabled() gc.disable() diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-02-00-34-42.bpo-31356.54Lb8U.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-02-00-34-42.bpo-31356.54Lb8U.rst new file mode 100644 index 000000000000..792f314c7a6d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-02-00-34-42.bpo-31356.54Lb8U.rst @@ -0,0 +1,3 @@ +Add a new contextmanager to the gc module that temporarily disables the GC +and restores the previous state. The implementation is done in C to assure +atomicity and speed. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 8ba1093c029d..c057d25a12b0 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -1067,6 +1067,10 @@ static PyObject * gc_enable_impl(PyObject *module) /*[clinic end generated code: output=45a427e9dce9155c input=81ac4940ca579707]*/ { + if(_PyRuntime.gc.disabled_threads){ + PyErr_WarnEx(PyExc_RuntimeWarning, "Garbage collector enabled while another " + "thread is inside gc.ensure_enabled",1); + } _PyRuntime.gc.enabled = 1; Py_RETURN_NONE; } @@ -1508,6 +1512,102 @@ static PyMethodDef GcMethods[] = { {NULL, NULL} /* Sentinel */ }; +typedef struct { + PyObject_HEAD + int previous_gc_state; +} ensure_disabled_object; + + +static void +ensure_disabled_object_dealloc(ensure_disabled_object *m_obj) +{ + Py_TYPE(m_obj)->tp_free((PyObject*)m_obj); +} + +static PyObject * +ensure_disabled__enter__method(ensure_disabled_object *self, PyObject *args) +{ + PyGILState_STATE gstate = PyGILState_Ensure(); + ++_PyRuntime.gc.disabled_threads; + self->previous_gc_state = _PyRuntime.gc.enabled; + gc_disable_impl(NULL); + PyGILState_Release(gstate); + Py_RETURN_NONE; +} + +static PyObject * +ensure_disabled__exit__method(ensure_disabled_object *self, PyObject *args) +{ + PyGILState_STATE gstate = PyGILState_Ensure(); + --_PyRuntime.gc.disabled_threads; + if(self->previous_gc_state){ + gc_enable_impl(NULL); + }else{ + gc_disable_impl(NULL); + } + PyGILState_Release(gstate); + Py_RETURN_NONE; +} + + + +static struct PyMethodDef ensure_disabled_object_methods[] = { + {"__enter__", (PyCFunction) ensure_disabled__enter__method, METH_NOARGS}, + {"__exit__", (PyCFunction) ensure_disabled__exit__method, METH_VARARGS}, + {NULL, NULL} /* sentinel */ +}; + +static PyObject * +new_disabled_obj(PyTypeObject *type, PyObject *args, PyObject *kwdict){ + ensure_disabled_object *self; + self = (ensure_disabled_object *)type->tp_alloc(type, 0); + return (PyObject *) self; +}; + +static PyTypeObject gc_ensure_disabled_type = { + PyVarObject_HEAD_INIT(NULL, 0) + "gc.ensure_disabled", /* tp_name */ + sizeof(ensure_disabled_object), /* tp_size */ + 0, /* tp_itemsize */ + /* methods */ + (destructor) ensure_disabled_object_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + PyObject_GenericGetAttr, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ensure_disabled_object_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + new_disabled_obj, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + + static struct PyModuleDef gcmodule = { PyModuleDef_HEAD_INIT, "gc", /* m_name */ @@ -1548,6 +1648,12 @@ PyInit_gc(void) if (PyModule_AddObject(m, "callbacks", _PyRuntime.gc.callbacks) < 0) return NULL; + if (PyType_Ready(&gc_ensure_disabled_type) < 0) + return NULL; + if (PyModule_AddObject(m, "ensure_disabled", (PyObject*) &gc_ensure_disabled_type) < 0) + return NULL; + + #define ADD_INT(NAME) if (PyModule_AddIntConstant(m, #NAME, NAME) < 0) return NULL ADD_INT(DEBUG_STATS); ADD_INT(DEBUG_COLLECTABLE);

1 0